Add the concept of a per-impl 'warning todo' which means the output is right but the warning is pending.

Author: chriseppstein

lib/sass_spec/annotate/cli.rb

@@ -141,6 +141,7 @@ def annotate
     return true
   end
 
+  # If you change this, also change TestCaseMetadata.merge_options
   def annotate_path(path)
     report(message: "#{path}:", complete: "") do
       options_file = path.end_with?("options.yml") ? path : File.join(path, "options.yml")

lib/sass_spec/test.rb

@@ -25,8 +25,13 @@ def run_spec_test(test_case, options = {})
   return unless handle_unexpected_error!(test_case, status, error, options)
   return unless handle_unexpected_pass!(test_case, status, error, options)
   return unless handle_output_difference!(test_case, options)
-  return unless handle_expected_error_message!(test_case, options)
-  return unless handle_unexpected_error_message!(test_case, options)
+
+  if test_case.warning_todo? && !options[:run_todo]
+    skip "Skipped warning check for #{test_case.folder}"
+  else
+    return unless handle_expected_error_message!(test_case, options)
+    return unless handle_unexpected_error_message!(test_case, options)
+  end
 end
 
 def interact(test_case, default, &block)
@@ -49,11 +54,18 @@ def handle_expected_error_message!(test_case, options)
     interact(test_case, :fail) do |i|
       i.prompt(error_msg.nil? ? "An error message was expected but wasn't produced." :
                                 "Error output doesn't match what was expected.")
-      i.choice(:show, "Show diff.") do
-        display_text_block(
-          Diffy::Diff.new("Expected\n" + expected_error_msg,
-                          "Actual\n" + error_msg).to_s(:color))
-        i.restart!
+      if error_msg.nil?
+        i.choice(:show, "Show expected error.") do
+          display_text_block(expected_error_msg)
+          i.restart!
+        end
+      else
+        i.choice(:show, "Show diff.") do
+          display_text_block(
+            Diffy::Diff.new("Expected\n#{expected_error_msg}",
+                            "Actual\n#{error_msg}").to_s(:color))
+          i.restart!
+        end
       end
 
       i.choice(:overwrite, "Update expected output and pass test.") do
@@ -66,6 +78,11 @@ def handle_expected_error_message!(test_case, options)
         return false
       end
 
+      i.choice(:todo, "Mark warning as todo for #{test_case.impl}.") do
+        change_options(test_case.options_path, add_warning_todo: [test_case.impl])
+        return false
+      end
+
       i.choice(:fail, "Mark as failed.")
 
       i.choice(:exit, "Exit testing.") do
@@ -112,6 +129,11 @@ def handle_unexpected_error_message!(test_case, options)
       return false
     end
 
+    i.choice(:todo, "Mark warning as todo for #{test_case.impl}.") do
+      change_options(test_case.options_path, add_warning_todo: [test_case.impl])
+      return false
+    end
+
     i.choice(:fail, "Mark as failed.")
 
     i.choice(:exit, "Exit testing.") do
@@ -406,12 +428,8 @@ def change_options(options_file, options)
                        {}
                      end
 
-  existing_options.update(options)
+  existing_options = SassSpec::TestCaseMetadata.merge_options(existing_options, options)
 
-  existing_options.each do |k, v|
-    existing_options.delete(k) if v.nil?
-  end
-  
   if existing_options.any?
     File.open(options_file, "w") do |f|
       f.write(existing_options.to_yaml)

lib/sass_spec/test_case.rb

@@ -79,8 +79,16 @@ def should_fail?
     @should_fail
   end
 
+  def impl
+    @options[:engine_adapter].describe
+  end
+
   def todo?
-    @metadata.todo?(@options[:engine_adapter].describe)
+    @metadata.todo?(impl)
+  end
+
+  def warning_todo?
+    @metadata.warning_todo?(impl)
   end
 
   def metadata

lib/sass_spec/test_case_metadata.rb

@@ -6,7 +6,36 @@ def self.cache
       @metadata_cache ||= {}
     end
 
-    ACCUMULATED_OPTIONS = [:todo, :expect_failure]
+    # If you change this, also change Annotate::CLI#annotate_path
+    def self.merge_options(existing_opts, new_opts)
+      existing_opts = existing_opts.dup
+
+      new_opts.each do |key, value|
+        if key =~ /add_(.*)/
+          key = $1.to_sym
+          existing_opts[key] ||= []
+          value.each do |v|
+            existing_opts[key] << v
+          end
+          existing_opts[key].uniq!
+        elsif key =~ /remove_(.*)/
+          key = $1.to_sym
+          existing_opts[key] ||= []
+          value.each do |v|
+            existing_opts[key].delete(v)
+          end
+          existing_opts.delete(key) if existing_opts[key].empty?
+        elsif value.nil?
+          existing_opts.delete(key)
+        else
+          existing_opts[key] = value
+        end
+      end
+
+      existing_opts
+    end
+
+    ACCUMULATED_OPTIONS = [:todo, :expect_failure, :warning_todo]
 
     attr_reader :options
 
@@ -47,6 +76,10 @@ def resolve_options(dir)
       self.class.cache[dir] ||= _resolve_options(dir).freeze
     end
 
+    def warning_todo?(impl)
+      @options[:warning_todo] && @options[:warning_todo].include?(impl)
+    end
+
     def todo?(impl)
       @options[:todo] && @options[:todo].include?(impl)
     end

spec/libsass-closed-issues/issue_1578/options.yml

@@ -0,0 +1,3 @@
+---
+:warning_todo:
+- libsass

spec/libsass-closed-issues/issue_308/options.yml

@@ -0,0 +1,3 @@
+---
+:warning_todo:
+- libsass

spec/output_styles/compact/libsass-closed-issues/issue_1578/options.yml

@@ -0,0 +1,3 @@
+---
+:warning_todo:
+- libsass

spec/output_styles/compact/libsass-closed-issues/issue_308/options.yml

@@ -0,0 +1,3 @@
+---
+:warning_todo:
+- libsass

spec/output_styles/compact/parser/operations/logic_eq/dimensions/pairs/options.yml

@@ -0,0 +1,3 @@
+---
+:warning_todo:
+- libsass

spec/output_styles/compact/parser/operations/logic_eq/mixed/pairs/options.yml

@@ -0,0 +1,3 @@
+---
+:warning_todo:
+- libsass