Merge pull request #7 from buildkite-plugins/handle-test-errors

Report on test errors as well

Author: toolmantim

ruby/bin/annotate

@@ -12,10 +12,11 @@ abort("#{junits_dir} does not exist") unless Dir.exist?(junits_dir)
 job_pattern = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN']
 job_pattern = '-(.*).xml' if !job_pattern || job_pattern.empty?
 
-class Failure < Struct.new(:name, :classname, :body, :job); end
+class Failure < Struct.new(:name, :classname, :body, :job, :type)
+end
 
 junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"))
-all_failures = []
+failures = []
 
 junit_report_files.each do |file|
   next if File.directory?(file)
@@ -29,31 +30,40 @@ junit_report_files.each do |file|
     name = testcase.attributes['name'].to_s
     classname = testcase.attributes['classname'].to_s
     testcase.elements.each("failure") do |failure|
-      all_failures << Failure.new(name, classname, failure.text.chomp.strip, job)
+      failures << Failure.new(name, classname, failure.text, job, :failure)
+    end
+    testcase.elements.each("error") do |error|
+      failures << Failure.new(name, classname, error.text, job, :error)
     end
   end
 end
 
 STDERR.puts "--- ❓ Checking failures"
 
-if all_failures.empty?
-  STDERR.puts "There were no failures 🙌"
+if failures.empty?
+  STDERR.puts "There were no failures/errors 🙌"
   exit 0
 else
-  STDERR.puts "There are #{all_failures.length} failures 😭"
+  STDERR.puts "There #{failures.length == 1 ? "is 1 failure/error" : "are #{failures.length} failures/errors" } 😭"
 end
 
 STDERR.puts "--- ✍️ Preparing annotation"
 
-puts "There were #{all_failures.length} failures:\n\n"
+failures_count = failures.select {|f| f.type == :failure }.length
+errors_count = failures.select {|f| f.type == :error }.length
+
+puts [
+  failures_count == 0 ? nil : (failures_count == 1 ? "1 failure" : "#{failures_count} failures"),
+  errors_count === 0 ? nil : (errors_count == 1 ? "1 error" : "#{errors_count} errors"),
+].compact.join(" and ") + ":\n\n"
 
-all_failures.each do |failure|
+failures.each do |failure|
   puts "<details>"
   puts "<summary><code>#{failure.name} in #{failure.classname}</code></summary>\n\n"
-  puts "<code><pre>#{failure.body}</pre></code>\n\n"
+  puts "<code><pre>#{failure.body.chomp.strip}</pre></code>\n\n"
   if failure.job
     puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
   end
   puts "</details>"
-  puts "" unless failure == all_failures.last
+  puts "" unless failure == failures.last
 end

ruby/tests/annotate_test.rb

@@ -3,28 +3,77 @@
 
 describe "Junit annotate plugin parser" do
   it "handles no failures" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/no-test-errors/")
+    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/")
 
     assert_equal <<~OUTPUT, output
       Parsing junit-1.xml
       Parsing junit-2.xml
       --- ❓ Checking failures
-      There were no failures 🙌
+      There were no failures/errors 🙌
     OUTPUT
 
     assert_equal 0, status.exitstatus
   end
 
   it "handles failures across multiple files" do
-    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/two-test-errors/")
+    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/two-test-failures/")
 
     assert_equal <<~OUTPUT, output
       Parsing junit-1.xml
       Parsing junit-2.xml
       --- ❓ Checking failures
-      There are 2 failures 😭
+      There are 2 failures/errors 😭
       --- ✍️ Preparing annotation
-      There were 2 failures:
+      2 failures:
+      
+      <details>
+      <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
+      
+      <code><pre>Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+      
+        expected: 250
+             got: 500
+      
+        (compared using eql?)
+      ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
+      ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
+      ./spec/support/log.rb:17:in `run'
+      ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'</pre></code>
+      
+      in <a href="#1">Job #1</a>
+      </details>
+      
+      <details>
+      <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ in spec.models.account_spec</code></summary>
+      
+      <code><pre>Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+      
+        expected: 700
+             got: 500
+      
+        (compared using eql?)
+      ./spec/models/account_spec.rb:78:in `block (3 levels) in <top (required)>'
+      ./spec/support/database.rb:16:in `block (2 levels) in <top (required)>'
+      ./spec/support/log.rb:17:in `run'
+      ./spec/support/log.rb:66:in `block (2 levels) in <top (required)>'</pre></code>
+      
+      in <a href="#2">Job #2</a>
+      </details>
+    OUTPUT
+
+    assert_equal 0, status.exitstatus
+  end
+
+  it "handles failures and errors across multiple files" do
+    output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/test-failure-and-error/")
+
+    assert_equal <<~OUTPUT, output
+      Parsing junit-1.xml
+      Parsing junit-2.xml
+      --- ❓ Checking failures
+      There are 2 failures/errors 😭
+      --- ✍️ Preparing annotation
+      1 failure and 1 error:
       
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -70,9 +119,9 @@
     assert_equal <<~OUTPUT, output
       Parsing junit-123-456-custom-pattern.xml
       --- ❓ Checking failures
-      There are 1 failures 😭
+      There is 1 failure/error 😭
       --- ✍️ Preparing annotation
-      There were 1 failures:
+      1 failure:
       
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>
@@ -102,9 +151,9 @@
       Parsing sub-dir/junit-1.xml
       Parsing sub-dir/junit-2.xml
       --- ❓ Checking failures
-      There are 2 failures 😭
+      There are 2 failures/errors 😭
       --- ✍️ Preparing annotation
-      There were 2 failures:
+      2 failures:
       
       <details>
       <summary><code>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default in spec.models.account_spec</code></summary>

ruby/tests/no-test-errors/junit-1.xml renamed to ruby/tests/no-test-failures/junit-1.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="rspec" tests="2" skipped="0" failures="0" errors="1" time="49.290713" timestamp="2018-04-18T23:29:42+00:00" hostname="626d214475e4">
+  <testcase classname="spec.models.account_spec" name="Account#maximum_jobs_added_by_pipeline_changer returns 500 if the account is ABC" file="./spec/models/account_spec.rb" time="0.020013"/>
+  <testcase classname="spec.models.account_spec" name="Account#maximum_jobs_added_by_pipeline_changer returns 250 by default" file="./spec/models/account_spec.rb" time="0.967127">
+    <error message=" expected: 250 got: 500 (compared using eql?) " type="RSpec::Expectations::ExpectationNotMetError">Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+
+  expected: 250
+       got: 500
+
+  (compared using eql?)
+./spec/models/account_spec.rb:78:in `block (3 levels) in &lt;top (required)&gt;&apos;
+./spec/support/database.rb:16:in `block (2 levels) in &lt;top (required)&gt;&apos;
+./spec/support/log.rb:17:in `run&apos;
+./spec/support/log.rb:66:in `block (2 levels) in &lt;top (required)&gt;&apos;</error>
+  </testcase>
+</testsuite>

ruby/tests/no-test-errors/junit-2.xml renamed to ruby/tests/no-test-failures/junit-2.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="rspec" tests="1" skipped="0" failures="1" errors="0" time="49.290713" timestamp="2018-04-18T23:29:42+00:00" hostname="626d214475e4">
+  <testcase classname="spec.models.account_spec" name="Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ" file="./spec/models/account_spec.rb" time="0.967127">
+    <failure message=" expected: 700 got: 500 (compared using eql?) " type="RSpec::Expectations::ExpectationNotMetError">Failure/Error: expect(account.maximum_jobs_added_by_pipeline_changer).to eql(250)
+
+  expected: 700
+       got: 500
+
+  (compared using eql?)
+./spec/models/account_spec.rb:78:in `block (3 levels) in &lt;top (required)&gt;&apos;
+./spec/support/database.rb:16:in `block (2 levels) in &lt;top (required)&gt;&apos;
+./spec/support/log.rb:17:in `run&apos;
+./spec/support/log.rb:66:in `block (2 levels) in &lt;top (required)&gt;&apos;</failure>
+  </testcase>
+</testsuite>