diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7d4dd8..8d9c0c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,10 +24,15 @@ jobs: - { name: cucumber1_3, bats: test/cucumber.bats } - { name: cucumber2_4, bats: test/cucumber.bats } - { name: minitest5, bats: test/minitest5.bats } + - { name: minitest6, bats: test/minitest6.bats } - { name: rspec3, bats: test/rspec3.bats } - { name: rspec4, bats: test/rspec4.bats } - { name: testunit, bats: test/testunit.bats } - { name: turnip, bats: test/turnip.bats } + exclude: + # Minitest 6 requires Ruby 3.2+ + - ruby: '2.7' + entry: { name: minitest6, bats: test/minitest6.bats } steps: - name: checkout diff --git a/Appraisals b/Appraisals index bbaa499..136792f 100644 --- a/Appraisals +++ b/Appraisals @@ -17,6 +17,11 @@ appraise 'minitest5' do gem 'minitest', '5.10.0' end +appraise 'minitest6' do + gem 'rake' + gem 'minitest', '~> 6.0' +end + appraise 'rspec3' do gem 'rspec', '~> 3.12' end diff --git a/gemfiles/minitest6.gemfile b/gemfiles/minitest6.gemfile new file mode 100644 index 0000000..2a2b718 --- /dev/null +++ b/gemfiles/minitest6.gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# This file was generated by Appraisal + +source 'https://rubygems.org' + +gem 'minitest', '~> 6.0' +gem 'rake' + +gemspec path: '../' diff --git a/lib/test_queue/runner/minitest.rb b/lib/test_queue/runner/minitest.rb index ff54a50..028c83b 100644 --- a/lib/test_queue/runner/minitest.rb +++ b/lib/test_queue/runner/minitest.rb @@ -2,9 +2,14 @@ require 'minitest' -raise 'requires Minitest version 5' unless Minitest::VERSION.to_i == 5 - -require_relative '../runner/minitest5' +case Minitest::VERSION.to_i +when 5 + require_relative '../runner/minitest5' +when 6 + require_relative '../runner/minitest6' +else + raise 'requires Minitest version 5 or 6' +end module TestQueue class Runner diff --git a/lib/test_queue/runner/minitest6.rb b/lib/test_queue/runner/minitest6.rb new file mode 100644 index 0000000..c2d16c3 --- /dev/null +++ b/lib/test_queue/runner/minitest6.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require_relative '../runner' + +module Minitest + def self.run_all_suites(reporter, options) + suites = Runnable.runnables + suites.map { |suite| suite.run_suite reporter, options } + end + + class Runnable + def failure_count + failures.length + end + end + + class Test + def self.runnables=(runnables) + @@runnables = runnables + end + + # Synchronize all tests, even serial ones. + # + # Minitest runs serial tests before parallel ones to ensure the + # unsynchronized serial tests don't overlap the parallel tests. But since + # the test-queue master hands out tests without actually loading their + # code, there's no way to know which are parallel and which are serial. + # Synchronizing serial tests does add some overhead, but hopefully this is + # outweighed by the speed benefits of using test-queue. + def _synchronize + Test.io_lock.synchronize { yield } + end + end + + class ProgressReporter + # Override original method to make test-queue specific output + def record(result) + io.print ' ' + io.print result.klass + io.print ': ' + io.print result.result_code + io.puts(' <%.3f>' % result.time) + end + end + + begin + require 'minitest/minitest_reporter_plugin' + + class << self + private + + def total_count(_options) + 0 + end + end + rescue LoadError + # noop + end +end + +module TestQueue + class Runner + class Minitest < Runner + def initialize + @options = ::Minitest.process_args ARGV + + if ::Minitest.respond_to?(:seed) + ::Minitest.seed = @options[:seed] + srand ::Minitest.seed + end + + if ::Minitest::Test.runnables.any? { |r| r.runnable_methods.any? } + raise 'Do not `require` test files. Pass them via ARGV instead and they will be required as needed.' + end + + super(TestFramework::Minitest.new) + end + + def start_master + puts "Run options: #{@options[:args]}\n\n" + + super + end + + def run_worker(iterator) + ::Minitest::Test.runnables = iterator + ::Minitest.run ? 0 : 1 + end + end + end + + class TestFramework + class Minitest < TestFramework + def all_suite_files + ARGV + end + + def suites_from_file(path) + ::Minitest::Test.reset + require File.absolute_path(path) + ::Minitest::Test.runnables + .reject { |s| s.runnable_methods.empty? } + .map { |s| [s.name, s] } + end + end + end +end diff --git a/test/examples/example_minispec6.rb b/test/examples/example_minispec6.rb new file mode 100644 index 0000000..b4629dc --- /dev/null +++ b/test/examples/example_minispec6.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'minitest/spec' + +class Meme + def i_can_has_cheezburger? + 'OHAI!' + end + + def will_it_blend? + 'YES!' + end +end + +describe Meme do + before do + @meme = Meme.new + end + + describe 'when asked about cheeseburgers' do + it 'must respond positively' do + sleep 0.1 + _(@meme.i_can_has_cheezburger?).must_equal 'OHAI!' + end + end + + describe 'when asked about blending possibilities' do + it "won't say no" do + sleep 0.1 + _(@meme.will_it_blend?).wont_match(/^no/i) + end + + if ENV['FAIL'] + it 'fails' do + _(@meme.will_it_blend?).must_equal 'NO!' + end + end + end +end diff --git a/test/examples/example_minitest6.rb b/test/examples/example_minitest6.rb new file mode 100644 index 0000000..a489bc3 --- /dev/null +++ b/test/examples/example_minitest6.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'minitest/autorun' + +class MinitestEqual < Minitest::Test + def test_equal + assert_equal 1, 1 + end +end + +30.times do |i| + Object.const_set("MinitestSleep#{i}", Class.new(Minitest::Test) do + define_method(:test_sleep) do + start = Time.now + sleep(0.25) + assert_in_delta Time.now - start, 0.25, 0.02 + end + end) +end + +if ENV['FAIL'] + class MinitestFailure < Minitest::Test + def test_fail + assert_equal 0, 1 + end + end +end + +if ENV['KILL'] + class MinitestKilledFailure < Minitest::Test + def test_kill + Process.kill(9, $$) + end + end +end diff --git a/test/minitest6.bats b/test/minitest6.bats new file mode 100644 index 0000000..f49457e --- /dev/null +++ b/test/minitest6.bats @@ -0,0 +1,209 @@ +load "testlib" + +SCRATCH=tmp/minitest6-tests + +setup() { + require_gem "minitest" "~> 6.0" + rm -rf $SCRATCH + mkdir -p $SCRATCH +} + +teardown() { + rm -rf $SCRATCH +} + +@test "minitest-queue (minitest6) succeeds when all tests pass" { + run bundle exec minitest-queue ./test/examples/*_minitest6.rb + assert_status 0 + assert_output_contains "Starting test-queue master" +} + +@test "minitest-queue (minitest6) succeeds when all tests pass with the --seed option" { + run bundle exec minitest-queue --seed 1234 ./test/examples/*_minitest6.rb + assert_status 0 + assert_output_contains "Run options: --seed 1234" + assert_output_contains "Starting test-queue master" +} + +@test "minitest-queue (minitest6) succeeds when all tests pass with the SEED env variable" { + export SEED=1234 + run bundle exec minitest-queue ./test/examples/*_minitest6.rb + assert_status 0 + assert_output_contains "Run options: --seed 1234" + assert_output_contains "Starting test-queue master" +} + +@test "minitest-queue (minitest6) fails when a test fails" { + export FAIL=1 + run bundle exec minitest-queue ./test/examples/*_minitest6.rb + assert_status 1 + assert_output_contains "Starting test-queue master" + assert_output_contains "1) Failure:" + assert_output_contains "MinitestFailure#test_fail" +} + +@test "TEST_QUEUE_FORCE allowlists certain tests" { + export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MinitestSleep11,MinitestSleep8" + run bundle exec minitest-queue ./test/examples/*_minitest6.rb + assert_status 0 + assert_output_contains "Starting test-queue master" + assert_output_contains "MinitestSleep11" + assert_output_contains "MinitestSleep8" + refute_output_contains "MinitestSleep9" +} + +assert_test_queue_force_ordering() { + run bundle exec minitest-queue "$@" + assert_status 0 + assert_output_contains "Starting test-queue master" + + # Turn the list of suites that were run into a comma-separated list. Input + # looks like: + # SuiteName: . <0.001> + actual_tests=$(echo "$output" | \ + egrep '^ .*: \.+ <' | \ + sed -E -e 's/^ (.*): \.+.*/\1/' | \ + tr '\n' ',' | \ + sed -e 's/,$//') + assert_equal "$TEST_QUEUE_FORCE" "$actual_tests" +} + +@test "TEST_QUEUE_FORCE ensures test ordering" { + export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="Meme::when asked about cheeseburgers,MinitestEqual" + + # Without stats file + rm -f .test_queue_stats + assert_test_queue_force_ordering ./test/examples/example_minitest6.rb ./test/examples/example_minispec6.rb + rm -f .test_queue_stats + assert_test_queue_force_ordering ./test/examples/example_minispec6.rb ./test/examples/example_minitest6.rb + + # With stats file + assert_test_queue_force_ordering ./test/examples/example_minitest6.rb ./test/examples/example_minispec6.rb + assert_test_queue_force_ordering ./test/examples/example_minispec6.rb ./test/examples/example_minitest6.rb +} + +@test "minitest-queue fails if TEST_QUEUE_FORCE specifies nonexistent tests" { + export TEST_QUEUE_WORKERS=1 TEST_QUEUE_FORCE="MinitestSleep11,DoesNotExist" + run bundle exec minitest-queue ./test/examples/*_minitest6.rb + assert_status 1 + assert_output_contains "Failed to discover DoesNotExist specified in TEST_QUEUE_FORCE" +} + +@test "multi-master central master succeeds when all tests pass" { + export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) + export SLEEP_AS_RELAY=1 + TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb || true & + sleep 0.1 + TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb + wait + + assert_status 0 + assert_output_contains "Starting test-queue master" +} + +@test "multi-master remote master succeeds when all tests pass" { + export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) + export SLEEP_AS_MASTER=1 + TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb || true & + sleep 0.1 + TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb + wait + + assert_status 0 + assert_output_contains "Starting test-queue master" +} + +@test "multi-master central master fails when a test fails" { + export FAIL=1 + export SLEEP_AS_RELAY=1 + export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) + TEST_QUEUE_RELAY=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb || true & + sleep 0.1 + TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb + wait + + assert_status 1 + assert_output_contains "Starting test-queue master" + assert_output_contains "1) Failure:" + assert_output_contains "MinitestFailure#test_fail" +} + +@test "multi-master remote master fails when a test fails" { + export FAIL=1 + export SLEEP_AS_MASTER=1 + export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) + TEST_QUEUE_SOCKET=0.0.0.0:12345 bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb || true & + sleep 0.1 + TEST_QUEUE_RELAY=0.0.0.0:12345 run bundle exec ruby ./test/sleepy_runner.rb ./test/examples/example_minitest6.rb + wait + + assert_status 1 + assert_output_contains "Starting test-queue master" + assert_output_contains "1) Failure:" + assert_output_contains "MinitestFailure#test_fail" +} + +@test "multi-master central master prints out remote master messages" { + export TEST_QUEUE_RELAY_TOKEN=$(date | cksum | cut -d' ' -f1) + TEST_QUEUE_RELAY=0.0.0.0:12345 TEST_QUEUE_REMOTE_MASTER_MESSAGE="hello from remote master" bundle exec minitest-queue ./test/examples/example_minitest6.rb & + TEST_QUEUE_SOCKET=0.0.0.0:12345 run bundle exec minitest-queue ./test/examples/example_minitest6.rb + wait + + assert_status 0 + assert_output_contains "hello from remote master" +} + +@test "recovers from child processes dying in an unorderly way" { + export KILL=1 + run bundle exec minitest-queue ./test/examples/example_minitest6.rb + assert_status 1 + assert_output_contains "SIGKILL (signal 9)" +} + +@test "minitest-queue fails when TEST_QUEUE_WORKERS is <= 0" { + export TEST_QUEUE_WORKERS=0 + run bundle exec minitest-queue ./test/examples/example_minitest6.rb + assert_status 1 + assert_output_contains "Worker count (0) must be greater than 0" +} + +@test "minitest-queue fails when given a missing test file" { + run bundle exec minitest-queue ./test/examples/does_not_exist.rb + assert_status 1 + assert_output_contains "Aborting: Discovering suites failed" +} + +@test "minitest-queue fails when given a malformed test file" { + [ -f README.md ] + run bundle exec minitest-queue README.md + assert_status 1 + assert_output_contains "Aborting: Discovering suites failed" +} + +@test "minitest-queue handles test file being deleted" { + cp test/examples/example_minitest6.rb test/examples/example_minispec6.rb $SCRATCH + + run bundle exec minitest-queue $SCRATCH/* + assert_status 0 + assert_output_contains "Meme::when asked about blending possibilities" + + rm $SCRATCH/example_minispec6.rb + + run bundle exec minitest-queue $SCRATCH/* + assert_status 0 + refute_output_contains "Meme::when asked about blending possibilities" +} + +@test "minitest-queue handles suites changing inside a file" { + cp test/examples/example_minispec6.rb $SCRATCH + + run bundle exec minitest-queue $SCRATCH/example_minispec6.rb + assert_status 0 + assert_output_contains "Meme::when asked about blending possibilities" + + sed -i'' -e 's/Meme/Meme2/g' $SCRATCH/example_minispec6.rb + + run bundle exec minitest-queue $SCRATCH/example_minispec6.rb + assert_status 0 + assert_output_contains "Meme2::when asked about blending possibilities" +}