Add mode to probe for passing todo tests

Author: mgreter

lib/sass_spec/cli.rb

@@ -68,6 +68,10 @@ def self.parse
         options[:run_todo] = true
       end
 
+      opts.on("--probe-todo", "Run and report tests marked as todo that unexpectedly pass. Defaults to false.") do
+        options[:probe_todo] = options[:run_todo] = true
+      end
+
       opts.on("--impl NAME", "Sets the name of the implementation being tested. Defaults to 'sass'") do |name|
         options[:implementation] = name
       end

lib/sass_spec/runner.rb

@@ -42,7 +42,18 @@ def run
       Minitest.reporter = Minitap::TapY
     end
 
-    Minitest.run(minioptions)
+    result = Minitest.run(minioptions)
+
+    if @options[:probe_todo]
+      puts "Reporting successfully probed todo tests"
+      test_cases.each do |test_case|
+        if test_case.todo? && test_case.result?
+          puts test_case.folder
+        end
+      end
+    end
+
+    result
   end
 
   def language_version

lib/sass_spec/test.rb

@@ -16,7 +16,7 @@ def run_spec_test(test_case, options = {})
 
   if test_case.overwrite?
     overwrite_test!(test_case)
-    return
+    return true
   end
 
   return unless handle_missing_output!(test_case)
@@ -30,6 +30,8 @@ def run_spec_test(test_case, options = {})
     return unless handle_expected_error_message!(test_case, options)
     return unless handle_unexpected_error_message!(test_case, options)
   end
+
+  return true
 end
 
 # if the test case is interactive it will do the interaction and return
@@ -51,6 +53,7 @@ def handle_expected_error_message!(test_case, options)
   expected_error_msg = _extract_error_message(test_case.expected_error, options)
 
   if expected_error_msg != error_msg
+    return false if test_case.probe_todo?
     interact(test_case, :fail) do |i|
       i.prompt(error_msg.nil? ? "An error message was expected but wasn't produced." :
                                 "Error output doesn't match what was expected.")
@@ -106,13 +109,14 @@ def handle_expected_error_message!(test_case, options)
 
 def handle_unexpected_error_message!(test_case, options)
   return true if test_case.verify_stderr?
-  
+
   _output, _clean_output, error, _status = test_case.output
 
   error_msg = _extract_error_message(error, options)
 
   return true if error_msg.nil? || error_msg.length == 0
 
+  return false if test_case.probe_todo?
   interact(test_case, :fail) do |i|
     i.prompt "Unexpected output to stderr"
 
@@ -160,6 +164,7 @@ def handle_output_difference!(test_case, options)
     return false
   end
 
+  return false if test_case.probe_todo?
   interact(test_case, :fail) do |i|
     i.prompt "output does not match expectation"
 
@@ -201,6 +206,7 @@ def handle_missing_output!(test_case)
   return true if File.exists?(test_case.expected_path)
 
   output, _, error, _ = test_case.output
+  return false if test_case.probe_todo?
   choice = interact(test_case, :fail) do |i|
     i.prompt "in #{test_case.name}\n" +
              "Expected output file does not exist."
@@ -253,6 +259,7 @@ def handle_unexpected_pass!(test_case, options)
       return false
     end
 
+    return false if test_case.probe_todo?
     choice = interact(test_case, :fail) do |i|
       i.prompt "In #{test_case.name}\n" +
                "A failure was expected but it compiled instead."
@@ -302,6 +309,7 @@ def handle_unexpected_error!(test_case, options)
       return false
     end
 
+    return false if test_case.probe_todo?
     choice = interact(test_case, :fail) do |i|
       i.prompt "In #{test_case.name}\n" +
                "An unexpected compiler error was encountered."
@@ -401,7 +409,7 @@ def migrate_test!(test_case, options)
   end
 
   start_version_index = SassSpec::LANGUAGE_VERSIONS.index(
-    test_case.metadata.start_version.to_s) 
+    test_case.metadata.start_version.to_s)
 
   if start_version_index >= current_version_index
     puts "Cannot migrate test. Test does not apply to an earlier version."
@@ -548,7 +556,7 @@ class SassSpec::Test < Minitest::Test
   def self.create_tests(test_cases, options = {})
     test_cases[0..options[:limit]].each do |test_case|
       define_method("test__#{test_case.name}") do
-        run_spec_test(test_case, options)
+        test_case.finalize(run_spec_test(test_case, options))
       end
     end
   end

lib/sass_spec/test_case.rb

@@ -11,6 +11,7 @@ def initialize(folder, options = {})
     @status_path = File.join(folder, "status")
     @options_path = File.join(folder, "options.yml")
     @options = options
+    @result = false
 
     # Probe filesystem once and cache the results
     @should_fail = File.file?(@status_path)
@@ -27,6 +28,14 @@ def find_input_path(folder)
     input_files.first
   end
 
+  def result?
+    @result
+  end
+
+  def finalize(result)
+    @result = result
+  end
+
   def folder
     @folder
   end
@@ -91,6 +100,11 @@ def todo?
     @metadata.todo?(impl)
   end
 
+  def probe_todo?
+    # run todo tests but do not fail runner if one does not pass
+    @options[:probe_todo] && (todo? || warning_todo?) && !interactive?
+  end
+
   def warning_todo?
     @metadata.warning_todo?(impl)
   end