Reduce Redis key size

Author: ChrisBr

ruby/lib/ci/queue/redis.rb

@@ -12,6 +12,7 @@
 require 'ci/queue/redis/test_time_record'
 require 'ci/queue/redis/test_duration_moving_averages'
 require 'ci/queue/redis/update_test_duration_moving_average'
+require 'ci/queue/redis/key_shortener'
 
 module CI
   module Queue

ruby/lib/ci/queue/redis/base.rb

@@ -102,7 +102,7 @@ def master_worker_id
         attr_reader :redis, :redis_url
 
         def key(*args)
-          ['build', build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
 
         def build_id

ruby/lib/ci/queue/redis/build_record.rb

@@ -124,7 +124,7 @@ def record_stats(stats, pipeline: redis)
         end
 
         def key(*args)
-          ['build', config.build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
       end
     end

ruby/lib/ci/queue/redis/grind_record.rb

@@ -52,7 +52,7 @@ def pop_warnings
         attr_reader :redis, :config
 
         def key(*args)
-          ['build', config.build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
 
         def record_stats(stats, pipeline: redis)

ruby/lib/ci/queue/redis/key_shortener.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+require 'digest/md5'
+
+module CI
+  module Queue
+    module Redis
+      module KeyShortener
+        # Suffix mapping for common key patterns
+        SUFFIX_ALIASES = {
+          'running' => 'r',
+          'processed' => 'p',
+          'queue' => 'q',
+          'owners' => 'o',
+          'error-reports' => 'e',
+          'requeues-count' => 'rc',
+          'assertions' => 'a',
+          'errors' => 'er',
+          'failures' => 'f',
+          'skips' => 's',
+          'requeues' => 'rq',
+          'total_time' => 't',
+          'test_failed_count' => 'fc',
+          'completed' => 'c',
+          'master-status' => 'm',
+          'created-at' => 'ca',
+          'workers' => 'w',
+          'worker' => 'w',
+          'warnings' => 'wn',
+          'worker-errors' => 'we',
+          'flaky-reports' => 'fl',
+        }.freeze
+
+        # We're transforming the key to a shorter format to minimize network traffic.
+        #
+        # Strategy:
+        # - Shorten prefix: 'b' instead of 'build'
+        # - Hash UUID: 8-char MD5 instead of 36-char UUID
+        # - Alias suffixes: single letters instead of full words
+        #
+        # Example:
+        #   build:unit:019aef0e-c010-433e-b706-c658d3c16372:running (55 bytes)
+        #   -> b:f03d3bef:r (13 bytes, 76% reduction)
+
+        def self.key(build_id, *args)
+          digest = Digest::MD5.hexdigest(build_id)[0..7]
+          shortened_args = args.map { |arg| SUFFIX_ALIASES[arg] || arg }
+
+          ['b', digest, *shortened_args].join(':')
+        end
+      end
+    end
+  end
+end

ruby/test/ci/queue/redis_test.rb

@@ -203,9 +203,9 @@ def test_acknowledge_returns_false_if_the_test_was_picked_up_by_another_worker
   end
 
   def test_workers_register
-    assert_equal 1, @redis.scard('build:42:workers')
+    assert_equal 1, @redis.scard(CI::Queue::Redis::KeyShortener.key('42', 'workers'))
     worker(2)
-    assert_equal 2, @redis.scard('build:42:workers')
+    assert_equal 2, @redis.scard(CI::Queue::Redis::KeyShortener.key('42', 'workers'))
   end
 
   def test_timeout_warning

ruby/test/integration/minitest_redis_test.rb

@@ -22,6 +22,130 @@ def setup
       @exe = File.expand_path('../../../exe/minitest-queue', __FILE__)
     end
 
+<<<<<<< HEAD
+=======
+    def test_default_reporter
+      out, err = capture_subprocess_io do
+        system(
+          { 'BUILDKITE' => '1' },
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', '1',
+          '--worker', '1',
+          '--timeout', '1',
+          '--max-requeues', '1',
+          '--requeue-tolerance', '1',
+          '-Itest',
+          'test/dummy_test.rb',
+          chdir: 'test/fixtures/',
+        )
+      end
+
+      assert_empty err
+      assert_match(/Expected false to be truthy/, normalize(out)) # failure output
+      result = normalize(out.lines.last.strip)
+      assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result
+    end
+
+    def test_lost_test_with_heartbeat_monitor
+      _, err = capture_subprocess_io do
+        2.times.map do |i|
+          Thread.start  do
+            system(
+              { 'BUILDKITE' => '1' },
+              @exe, 'run',
+              '--queue', @redis_url,
+              '--seed', 'foobar',
+              '--build', '1',
+              '--worker', i.to_s,
+              '--timeout', '1',
+              '--max-requeues', '1',
+              '--requeue-tolerance', '1',
+              '--heartbeat', '1',
+              '-Itest',
+              'test/lost_test.rb',
+              chdir: 'test/fixtures/',
+            )
+          end
+        end.each(&:join)
+      end
+
+      assert_empty err
+
+      Tempfile.open('warnings') do |warnings_file|
+        out, err = capture_subprocess_io do
+          system(
+            @exe, 'report',
+            '--queue', @redis_url,
+            '--build', '1',
+            '--timeout', '1',
+            '--warnings-file', warnings_file.path,
+            '--heartbeat',
+            chdir: 'test/fixtures/',
+            )
+        end
+
+        assert_empty err
+        result = normalize(out.lines[1].strip)
+        assert_equal "Ran 1 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)", result
+        warnings = JSON.parse(warnings_file.read)
+        assert_equal 1, warnings.size
+      end
+    end
+
+    def test_verbose_reporter
+      out, err = capture_subprocess_io do
+        system(
+          { 'BUILDKITE' => '1' },
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', '1',
+          '--worker', '1',
+          '--timeout', '1',
+          '--max-requeues', '1',
+          '--requeue-tolerance', '1',
+          '-Itest',
+          'test/dummy_test.rb',
+          '-v',
+          chdir: 'test/fixtures/',
+        )
+      end
+
+      assert_empty err
+      assert_match(/ATest#test_foo \d+\.\d+ = S/, normalize(out)) # verbose test ouptut
+      result = normalize(out.lines.last.strip)
+      assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result
+    end
+
+    def test_debug_log
+      Tempfile.open('debug_log') do |log_file|
+        out, err = capture_subprocess_io do
+          system(
+            { 'BUILDKITE' => '1' },
+            @exe, 'run',
+            '--queue', @redis_url,
+            '--seed', 'foobar',
+            '--build', '1',
+            '--worker', '1',
+            '--timeout', '1',
+            '--max-requeues', '1',
+            '--requeue-tolerance', '1',
+            '-Itest',
+            'test/dummy_test.rb',
+            '--debug-log', log_file.path,
+            chdir: 'test/fixtures/',
+          )
+        end
+
+      assert_includes File.read(log_file.path), 'INFO -- : Finished \'["exists", "b:c4ca4238:w:1:q"]\': 0'
+      assert_empty err
+      result = normalize(out.lines.last.strip)
+      assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result
+      end
+    end
+
     def test_buildkite_output
       out, err = capture_subprocess_io do
         system(
@@ -240,7 +364,7 @@ def test_retry_fails_when_test_run_is_expired
       assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output
 
       one_day = 60 * 60 * 24
-      key = ['build', "1", "created-at"].join(':')
+      key = CI::Queue::Redis::KeyShortener.key("1", "created-at")
       @redis.set(key, Time.now - one_day)
 
       out, err = capture_subprocess_io do