Merge pull request #191 from Shopify/cbruckmayer/fix-retry

Fix created_at race condition

Author: ChrisBr

ruby/lib/ci/queue/redis/base.rb

@@ -22,14 +22,18 @@ def exhausted?
         end
 
         def expired?
-          if (created_at = redis.get(key('master-created-at')))
+          if (created_at = redis.get(key('created-at')))
             (created_at.to_f + config.redis_ttl + TEN_MINUTES) < Time.now.to_f
           else
             # if there is no created at set anymore we assume queue is expired
             true
           end
         end
 
+        def created_at=(timestamp)
+          redis.setnx(key('created-at'), timestamp)
+        end
+
         def size
           redis.multi do |transaction|
             transaction.llen(key('queue'))
@@ -71,6 +75,10 @@ def queue_initialized?
           end
         end
 
+        def queue_initializing?
+          master_status == 'setup'
+        end
+
         def increment_test_failed
           redis.incr(key('test_failed_count'))
         end

ruby/lib/ci/queue/redis/worker.rb

@@ -202,12 +202,10 @@ def push(tests)
               transaction.lpush(key('queue'), tests) unless tests.empty?
               transaction.set(key('total'), @total)
               transaction.set(key('master-status'), 'ready')
-              transaction.set(key('master-created-at'), Time.now.to_f)
 
               transaction.expire(key('queue'), config.redis_ttl)
               transaction.expire(key('total'), config.redis_ttl)
               transaction.expire(key('master-status'), config.redis_ttl)
-              transaction.expire(key('master-created-at'), config.redis_ttl)
             end
           end
           register

ruby/lib/ci/queue/static.rb

@@ -11,6 +11,8 @@ def from_uri(uri, config)
         end
       end
 
+      TEN_MINUTES = 60 * 10
+
       attr_reader :progress, :total
 
       def initialize(tests, config)
@@ -37,6 +39,14 @@ def populate(tests, random: nil)
         self
       end
 
+      def created_at=(timestamp)
+        @created_at ||= timestamp
+      end
+
+      def expired?
+        (@created_at.to_f TEN_MINUTES) < Time.now.to_f
+      end
+
       def populated?
         !!defined?(@index)
       end

ruby/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.24.0'
+    VERSION = '0.24.1'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

ruby/lib/minitest/queue/runner.rb

@@ -61,6 +61,8 @@ def run_command
           end
         end
 
+        queue.rescue_connection_errors { queue.created_at = Time.now.to_f }
+
         set_load_path
         Minitest.queue = queue
         reporters = [

ruby/test/integration/minitest_redis_test.rb

@@ -238,7 +238,7 @@ def test_retry_fails_when_test_run_is_expired
       assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output
 
       one_day = 60 * 60 * 24
-      key = ['build', "1", "master-created-at"].join(':')
+      key = ['build', "1", "created-at"].join(':')
       @redis.set(key, Time.now - one_day)
 
       out, err = capture_subprocess_io do