Switch to redis-ttl and use CI_QUEUE_REDIS_TTL as an env to set defaults

Author: Doug Edey

README.md

@@ -40,6 +40,7 @@ Two implementations are provided, please refer to the respective documentations:
 ## Redis Requirements
 
 `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/Shopify/ci-queue.

ruby/README.md

@@ -75,4 +75,4 @@ Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)
 
 `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
 
-You can also use `--report-expires-in` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)
+You can also use `--redis-ttl` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)

ruby/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ class Configuration
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed, :report_expires_in
+      attr_accessor :max_test_failed, :redis_ttl
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout
@@ -18,6 +18,7 @@ def from_env(env)
             seed: env['CIRCLE_SHA1'] || env['BUILDKITE_COMMIT'] || env['TRAVIS_COMMIT'] || env['HEROKU_TEST_RUN_COMMIT_VERSION'] || env['SEMAPHORE_GIT_SHA'],
             flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
             statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
+            redis_ttl: env['CI_QUEUE_REDIS_TTL'],
           )
         end
 
@@ -34,7 +35,7 @@ def initialize(
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
-        queue_init_timeout: nil, report_expires_in: 8 * 60 * 60
+        queue_init_timeout: nil, redis_ttl: 8 * 60 * 60
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -55,7 +56,7 @@ def initialize(
         @worker_id = worker_id
         self.max_consecutive_failures = max_consecutive_failures
         self.max_duration = max_duration
-        @report_expires_in = report_expires_in
+        @redis_ttl = redis_ttl
       end
 
       def queue_init_timeout

ruby/lib/ci/queue/redis/build_record.rb

@@ -41,7 +41,7 @@ def record_error(id, payload, stats: nil)
               id.dup.force_encoding(Encoding::BINARY),
               payload.dup.force_encoding(Encoding::BINARY),
             )
-            pipeline.expire(key('error-reports'), config.report_expires_in)
+            pipeline.expire(key('error-reports'), config.redis_ttl)
             record_stats(stats, pipeline: pipeline)
           end
           nil
@@ -91,7 +91,7 @@ def record_stats(stats, pipeline: redis)
           return unless stats
           stats.each do |stat_name, stat_value|
             pipeline.hset(key(stat_name), config.worker_id, stat_value)
-            pipeline.expire(key(stat-name), config.report_expires_in)
+            pipeline.expire(key(stat-name), config.redis_ttl)
           end
         end
 

ruby/lib/ci/queue/redis/grind_record.rb

@@ -16,7 +16,7 @@ def record_error(payload, stats: nil)
               key('error-reports'),
               payload.force_encoding(Encoding::BINARY),
             )
-            pipeline.expire(key('error-reports'), config.report_expires_in)
+            pipeline.expire(key('error-reports'), config.redis_ttl)
             record_stats(stats, pipeline: pipeline)
           end
           nil
@@ -59,7 +59,7 @@ def record_stats(stats, pipeline: redis)
           return unless stats
           stats.each do |stat_name, stat_value|
             pipeline.hset(key(stat_name), config.worker_id, stat_value)
-            pipeline.expire(key(stat_name), config.report_expires_in)
+            pipeline.expire(key(stat_name), config.redis_ttl)
           end
         end
       end

ruby/lib/ci/queue/redis/test_time_record.rb

@@ -24,7 +24,7 @@ def record_test_time(test_name, duration)
               test_time_key(test_name),
               duration.to_s.force_encoding(Encoding::BINARY),
             )
-            pipeline.expire(test_time_key(test_name), config.report_expires_in)
+            pipeline.expire(test_time_key(test_name), config.redis_ttl)
           end
           nil
         end
@@ -35,7 +35,7 @@ def record_test_name(test_name)
               all_test_names_key,
               test_name.dup.force_encoding(Encoding::BINARY),
             )
-            pipeline.expire(all_test_names_key, config.report_expires_in)
+            pipeline.expire(all_test_names_key, config.redis_ttl)
           end
           nil
         end

ruby/lib/ci/queue/redis/worker.rb

@@ -54,8 +54,8 @@ def poll
             end
           end
           redis.pipelined do |pipeline|
-            pipeline.expire(key('worker', worker_id, 'queue'), config.report_expires_in)
-            pipeline.expire(key('processed'), config.report_expires_in)
+            pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl)
+            pipeline.expire(key('processed'), config.redis_ttl)
           end
         rescue *CONNECTION_ERRORS
         end
@@ -203,13 +203,13 @@ def push(tests)
               transaction.set(key('total'), @total)
               transaction.set(key('master-status'), 'ready')
 
-              transaction.expire(key('queue'), config.report_expires_in)
-              transaction.expire(key('total'), config.report_expires_in)
-              transaction.expire(key('master-status'), config.report_expires_in)
+              transaction.expire(key('queue'), config.redis_ttl)
+              transaction.expire(key('total'), config.redis_ttl)
+              transaction.expire(key('master-status'), config.redis_ttl)
             end
           end
           register
-          redis.expire(key('workers'), config.report_expires_in)
+          redis.expire(key('workers'), config.redis_ttl)
         rescue *CONNECTION_ERRORS
           raise if @master
         end

ruby/lib/minitest/queue/runner.rb

@@ -491,8 +491,8 @@ def parser
             Defines how long the test report remain after the test run, in seconds.
             Defaults to 28,800 (8 hours)
           EOS
-          opts.on("--report-expires-in SECONDS", Integer, help) do |time|
-            queue.config.report_expires_in = time
+          opts.on("--redis-ttl SECONDS", Integer, help) do |time|
+            queue.config.redis_ttl = time
           end
 
           opts.separator ""

ruby/lib/rspec/queue.rb

@@ -162,8 +162,8 @@ def parser(options)
           Defaults to 28,800 (8 hours)
         EOS
         parsed.separator ""
-        parser.on("--report-expires-in SECONDS", Integer, help) do |time|
-          queue.config.report_expires_in = time
+        parser.on("--redis-ttl SECONDS", Integer, help) do |time|
+          queue.config.redis_ttl = time
         end
 
         parser