Merge pull request #692 from mgreter/feature/spec-runner-improve

Improve spec test runner

Author: mgreter

lib/sass_spec/engine_adapter.rb

@@ -1,5 +1,3 @@
-require "open3"
-
 class EngineAdapter
   def describe
     not_implemented
@@ -39,13 +37,15 @@ def describe
   end
 
   def version
-    stdout, stderr, status = Open3.capture3("#{@command} -v")
+    require 'open3'
+    stdout, stderr, status = Open3.capture3("#{@command} -v", :binmode => true)
     stdout.to_s
   end
 
 
   def compile(sass_filename, style)
-    stdout, stderr, status = Open3.capture3("#{@command} -t #{style} #{sass_filename}")
+    require 'open3'
+    stdout, stderr, status = Open3.capture3("#{@command} -t #{style} #{sass_filename}", :binmode => true)
     [stdout, stderr, status.exitstatus]
   end
 end
@@ -66,20 +66,34 @@ def version
 
   def compile(sass_filename, style)
     require 'sass'
+    # overloads STDERR
+    stderr = StringIO.new
+    # restore previous default encoding
+    encoding = Encoding.default_external
+    # Does not work as expected when tests are run in parallel
+    # It runs tests in threads, so stderr is shared across them
+    old_stderr, $stderr = $stderr, stderr
     begin
-      captured_stderr = StringIO.new
-      # Does not work as expected when tests are run in parallel
-      real_stderr, $stderr = $stderr, captured_stderr
-      begin
-        css_output = Sass.compile_file(sass_filename.to_s, :style => style.to_sym)
-        [css_output, captured_stderr.string, 0]
-      rescue Sass::SyntaxError => e
-        ["", "Error: " + e.message.to_s, 1]
-      rescue => e
-        ["", e.to_s, 2]
-      end
-    ensure
-      $stderr = real_stderr
+      Encoding.default_external = "UTF-8"
+      css_output = Sass.compile_file(sass_filename.to_s, :style => style.to_sym)
+      # strings come back as utf8 encoded
+      # internaly we only work with bytes
+      err_output = stderr.string
+      err_output.force_encoding('ASCII-8BIT')
+      css_output.force_encoding('ASCII-8BIT')
+      [css_output, err_output, 0]
+    rescue Sass::SyntaxError => e
+      # prepend the prefix to the message
+      # and indent all lines to match it
+      err_output = "Error: " + e.message.to_s
+                   .gsub(/(?:\r?\n)(?!\z)/, "\n       ")
+      err_output.force_encoding('ASCII-8BIT')
+      ["", err_output, 65]
+    rescue => e
+      ["", e.to_s, 2]
     end
+  ensure
+    $stderr = old_stderr
+    Encoding.default_external = encoding
   end
 end

lib/sass_spec/test.rb

@@ -11,20 +11,24 @@ def run_spec_test(test_case, options = {})
 
   if test_case.overwrite?
     if status != 0
-      File.open(test_case.status_path, "w+") do |f|
+      File.open(test_case.status_path, "w+", :binmode => true) do |f|
         f.write(status)
         f.close
       end
+    elsif (File.file?(test_case.status_path))
+      File.unlink(test_case.status_path)
     end
 
     if error.length > 0
-      File.open(test_case.error_path, "w+") do |f|
+      File.open(test_case.error_path, "w+", :binmode => true) do |f|
         f.write(error)
         f.close
       end
+    elsif (File.file?(test_case.error_path))
+      File.unlink(test_case.error_path)
     end
 
-    File.open(test_case.expected_path, "w+") do |f|
+    File.open(test_case.expected_path, "w+", :binmode => true) do |f|
       f.write(output)
       f.close
     end
@@ -43,15 +47,74 @@ def run_spec_test(test_case, options = {})
     if test_case.verify_stderr?
       # Compare only first line of error output (we can't compare stacktraces etc.)
       begin
+        skip = false
         error_lines = error.each_line
-        error_msg = error_lines.next.rstrip
-        if (error_msg =~ /DEPRECATION WARNING:? on line/)
-          error_msg = error_lines.next.rstrip
+        while error_line = error_lines.next do
+          if (error_line =~ /DEPRECATION WARNING/)
+            skip = false
+          end
+          if (error_line =~ /Error:/)
+            skip = false
+          end
+          # disable once we support this deprecation fully
+          if (error_line =~ /interpolation near operators will be simplified/)
+            skip = true
+            next
+          end
+          # disable once we support this deprecation fully
+          # if (error_line =~ /The subject selector operator \"!\" is deprecated and will be removed/)
+          #   skip = true
+          #   next
+          # end
+          # disable once we support this deprecation fully
+          if (error_line =~ /Passing a percentage as the alpha/)
+            skip = true
+            next
+          end
+          # disable once we support this deprecation fully (partial now)
+          # if (error_line =~ /, a non-string value, to unquote()/)
+          #   skip = true
+          #   next
+          # end
+          if (skip)
+            next
+          end
+          error_msg = error_line.rstrip
+          break
         end
         expected_error_lines = test_case.expected_error.each_line
-        expected_error_msg = expected_error_lines.next.rstrip
-        if (expected_error_msg =~ /DEPRECATION WARNING:? on line/)
-          expected_error_msg = expected_error_lines.next.rstrip
+        while expected_error_line = expected_error_lines.next do
+          if (expected_error_line =~ /DEPRECATION WARNING/)
+            skip = false
+          end
+          if (expected_error_line =~ /Error:/)
+            skip = false
+          end
+          # disable once we support this deprecation fully
+          if (expected_error_line =~ /interpolation near operators will be simplified/)
+            skip = true
+            next
+          end
+          # disable once we support this deprecation fully
+          # if (expected_error_line =~ /The subject selector operator \"!\" is deprecated and will be removed/)
+          #   skip = true
+          #   next
+          # end
+          # disable once we support this deprecation fully
+          if (expected_error_line =~ /Passing a percentage as the alpha/)
+            skip = true
+            next
+          end
+          # disable once we support this deprecation fully (partial now)
+          # if (expected_error_line =~ /, a non-string value, to unquote()/)
+          #   skip = true
+          #   next
+          # end
+          if (skip)
+            next
+          end
+          expected_error_msg = expected_error_line.rstrip
+          break
         end
         error_msg = _clean_debug_path(error_msg)
         expected_error_msg = _clean_debug_path(expected_error_msg)
@@ -79,12 +142,12 @@ def _clean_debug_path(error)
   pwd = Dir.pwd
   url = pwd.gsub(/\\/, '\/')
   error.gsub(/^.*?(input.scss:\d+ DEBUG:)/, '\1')
-       .gsub(/[ 	]+/, " ")
+       .gsub(/\/+/, "/")
        .gsub(/#{Regexp.quote(url)}\//, "/sass/sass-spec/")
        .gsub(/#{Regexp.quote(pwd)}\//, "/sass/sass-spec/")
        .gsub(/(?:\/todo_|_todo\/)/, "/")
-       .gsub(/\/libsass\-[a-z]+\-test\//, "/")
-       .gsub(/\/libsass\-[a-z]+\-issue/, "/libsass-issue")
+       .gsub(/\/libsass\-[a-z]+\-tests\//, "/")
+       .gsub(/\/libsass\-[a-z]+\-issues/, "/libsass-issues")
        .strip
 end
 

lib/sass_spec/test_case.rb

@@ -67,15 +67,24 @@ def output
     stdout, stderr, status = engine.compile(@input_path, @output_style)
 
     if @clean_test
-      cleaned = _clean_output(stdout)
+      clean_out = _clean_output(stdout)
     else
-      cleaned = _norm_output(stdout)
+      clean_out = _norm_output(stdout)
     end
-    @output ||= [stdout, cleaned, stderr, status]
+
+    stderr = _clean_error(stderr)
+    # always replace windows linefeeds
+    stdout = stdout.gsub(/(\r\n)/, "\n")
+    stderr = stderr.gsub(/(\r\n)/, "\n")
+
+    @output ||= [stdout, clean_out, stderr, status]
   end
 
   def expected
-    output = File.read(@expected_path, :encoding => "utf-8")
+    output = File.read(@expected_path, :binmode => true)
+    # we seem to get CP850 otherwise
+    # this provokes equal test to fail
+    output.force_encoding('ASCII-8BIT')
     if @clean_test
       @expected ||= _clean_output(output)
     else
@@ -84,12 +93,12 @@ def expected
   end
 
   def expected_error
-    @expected_error = _clean_error(File.read(@error_path, :encoding => "utf-8"))
+    @expected_error = _clean_error(File.read(@error_path, :binmode => true))
   end
 
   def expected_status
     if should_fail?
-      @expected_status = File.read(@status_path, :encoding => "utf-8").to_i
+      @expected_status = File.read(@status_path).to_i
     else
       @expected_status = 0
     end
@@ -99,35 +108,39 @@ def engine
     @options[:engine_adapter]
   end
 
+  # normalization happens for every test
   def _norm_output(css)
-    css = css.force_encoding('iso-8859-1').encode('utf-8')
-    css.gsub(/(?:\r?\n)+/, "\n")
-       .strip
+    # we dont want to test for linux or windows line-feeds
+    # but make sure we do not remove single cariage returns
+    css = css.gsub(/(?:\r?\n)+/, "\n")
+  end
+  def _norm_error(err)
+    # we dont want to test for linux or windows line-feeds
+    # but make sure we do not remove single cariage returns
+    err = err.gsub(/(?:\r?\n)+/, "\n")
+    # remove all newlines at the end of the file
+    err = err.sub(/(?:\r?\n)+\z/, "")
   end
 
+  # cleaning only happens when requested for test
+  # done by creating `expected.type.clean` flag file
   def _clean_output(css)
-    css = css.force_encoding('iso-8859-1').encode('utf-8')
-    css.gsub(/\s+/, " ")
-       .gsub(/ *\{/, " {\n")
+    css.gsub(/ *\{/, " {\n")
        .gsub(/([;,]) */, "\\1\n")
        .gsub(/ *\} */, " }\n")
        .gsub(/;(?:\s*;)+/m, ";")
        .gsub(/;\r?\n }/m, " }")
+       .gsub(/\r?\n/, "\n")
+       .sub(/(?:\r?\n)+\z/, "")
        .strip
   end
-
   def _clean_error(err)
-    pwd = Dir.pwd
-    url = pwd.gsub(/\\/, '/')
-    err = err.force_encoding('iso-8859-1').encode('utf-8')
-    err.gsub(/^.*?(input.scss:\d+ DEBUG:)/, '\1')
-       .gsub(/[ 	]+/, " ")
-       .gsub(/#{Regexp.quote(url)}\//, "/sass/sass-spec/")
-       .gsub(/#{Regexp.quote(pwd)}\//, "/sass/sass-spec/")
-       .gsub(/(?:\/todo_|_todo\/)/, "/")
-       .gsub(/\/libsass\-[a-z]+\-test\//, "/")
-       .gsub(/\/libsass\-[a-z]+\-issue/, "/libsass-issue")
-       .strip
+    err.gsub(/(?:\/todo_|_todo\/)/, "/") # hide todo pre/suffix
+       .gsub(/\/libsass\-[a-z]+\-tests\//, "/") # hide test directory
+       .gsub(/\/libsass\-[a-z]+\-issues\//, "/libsass-issues/") # normalize issue specs
+       .gsub(/[\w\/\-\\:]+?[\/\\]spec[\/\\]+/, "/sass/spec/") # normalize abs paths
+       .sub(/(?:\r?\n)*\z/, "\n") # make sure we have exactly one trailing linefeed
+       .sub(/\A(?:\r?[\n\s])+\z/, "") # clear the whole file if only whitespace
   end
 
 end

spec/colors/basic/error

@@ -1,6 +1,6 @@
 DEPRECATION WARNING: Passing red, a non-string value, to unquote()
 will be an error in future versions of Sass.
-        on line 8 of /sass/sass-spec/spec/colors/basic/input.scss
+        on line 8 of /sass/spec/colors/basic/input.scss
 DEPRECATION WARNING: Passing 0xf00, a non-string value, to unquote()
 will be an error in future versions of Sass.
-        on line 9 of /sass/sass-spec/spec/colors/basic/input.scss
+        on line 9 of /sass/spec/colors/basic/input.scss

spec/errors/extend/placeholder/missing/error

@@ -1,5 +1,5 @@
 Error: ".baz" failed to @extend "%foo".
        The selector "%foo" was not found.
        Use "@extend %foo !optional" if the extend should be able to fail.
-        on line 2 of /sass/sass-spec/spec/errors/extend/placeholder/missing/input.scss
+        on line 2 of /sass/spec/errors/extend/placeholder/missing/input.scss
   Use --trace for backtrace.

spec/errors/extend/selector/missing/error

@@ -1,5 +1,5 @@
 Error: ".baz" failed to @extend ".foo".
        The selector ".foo" was not found.
        Use "@extend .foo !optional" if the extend should be able to fail.
-        on line 2 of /sass/sass-spec/spec/errors/extend/selector/missing/input.scss
+        on line 2 of /sass/spec/errors/extend/selector/missing/input.scss
   Use --trace for backtrace.

spec/errors/fn-debug/property/error

@@ -1,3 +1,3 @@
 Error: Illegal nesting: Only properties may be nested beneath properties.
-        on line 3 of /sass/sass-spec/spec/errors/fn-debug/property/input.scss
+        on line 3 of /sass/spec/errors/fn-debug/property/input.scss
   Use --trace for backtrace.

spec/errors/fn-debug/ruleset/error

@@ -1 +1 @@
-/sass/sass-spec/spec/errors/fn-debug/ruleset/input.scss:2 DEBUG: debug
+/sass/spec/errors/fn-debug/ruleset/input.scss:2 DEBUG: debug

spec/errors/fn-debug/simple/error

@@ -1 +1 @@
-/sass/sass-spec/spec/errors/fn-debug/simple/input.scss:1 DEBUG: debug
+/sass/spec/errors/fn-debug/simple/input.scss:1 DEBUG: debug

spec/errors/fn-error/property/error

@@ -1,3 +1,3 @@
 Error: Illegal nesting: Only properties may be nested beneath properties.
-        on line 3 of /sass/sass-spec/spec/errors/fn-error/property/input.scss
+        on line 3 of /sass/spec/errors/fn-error/property/input.scss
   Use --trace for backtrace.