Merge pull request #144 from jvdp/report-slowest

Report specified amount of slowest tests (0 by default)

Author: jj-bk

README.md

@@ -56,6 +56,11 @@ Default: `junit`
 
 The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline.
 
+### `report-slowest` (optional)
+Default: `0`
+
+Include the specified number of slowest tests in the annotation. The annotation will always be shown.
+
 ## Developing
 
 To test the plugin hooks (in Bash) and the junit parser (in Ruby):

hooks/command

@@ -10,6 +10,8 @@ echo "--- :junit: Download the junits"
 artifacts_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX")"
 annotation_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-annotation-tmp.XXXXXXXXXX")"
 annotation_path="${annotation_dir}/annotation.md"
+annotation_style="info"
+fail_build=0
 
 function cleanup {
   rm -rf "${artifacts_dir}"
@@ -30,6 +32,7 @@ buildkite-agent artifact download \
 
 echo "--- :junit: Processing the junits"
 
+set +e
 docker \
   --log-level "error" \
   run \
@@ -38,25 +41,35 @@ docker \
     --volume "$PLUGIN_DIR/ruby:/src" \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN:-}" \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT:-}" \
+    --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST:-}" \
     ruby:2.7-alpine ruby /src/bin/annotate /junits \
       > "$annotation_path"
 
+if [[ $? -eq 64 ]]; then # special exit code to signal test failures
+  annotation_style="error"
+  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR:-false}" =~ (true|on|1) ]]; then
+    fail_build=1
+  fi
+fi
+
+set -e
+
 cat "$annotation_path"
 
 if grep -q "<details>" "$annotation_path"; then
-
   if ! check_size; then
     echo "--- :warning: Failures too large to annotate"
     echo "The failures are too large to create a build annotation. Please inspect the failed JUnit artifacts manually."
   else
     echo "--- :buildkite: Creating annotation"
     # shellcheck disable=SC2002
-    cat "$annotation_path" | buildkite-agent annotate --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT:-junit}" --style error
+    cat "$annotation_path" | buildkite-agent annotate --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT:-junit}" --style "$annotation_style"
   fi
+fi
 
-  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR:-false}" =~ (true|on|1) ]]
-  then
-    echo "--- :boom: Failing build due to error"
-    exit 1
-  fi
+if ((fail_build)); then
+  echo "--- :boom: Failing build due to error"
+  exit 1
+else
+  exit 0
 fi

plugin.yml

@@ -18,6 +18,8 @@ configuration:
       type: boolean
     context:
       type: string
+    report-slowest:
+      type: integer
   required:
     - artifacts
   additionalProperties: false

ruby/bin/annotate

@@ -17,12 +17,18 @@ job_pattern = '-(.*).xml' if !job_pattern || job_pattern.empty?
 failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT']
 failure_format = 'classname' if !failure_format || failure_format.empty?
 
-class Failure < Struct.new(:name, :failed_test, :body, :job, :type, :message)
+report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST'].to_i
+
+class Failure < Struct.new(:name, :unit_name, :body, :job, :type, :message)
+end
+
+class Timing < Struct.new(:name, :unit_name, :time)
 end
 
 junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"))
 testcases = 0
 failures = []
+timings = []
 
 def text_content(element)
   # Handle mulptiple CDATA/text children elements
@@ -55,48 +61,65 @@ junit_report_files.sort.each do |file|
   REXML::XPath.each(doc, '//testsuite/testcase') do |testcase|
     testcases += 1
     name = testcase.attributes['name'].to_s
-    failed_test = testcase.attributes[failure_format].to_s
+    unit_name = testcase.attributes[failure_format].to_s
+    time = testcase.attributes['time'].to_f
+    timings << Timing.new(name, unit_name, time)
     testcase.elements.each("failure") do |failure|
-      failures << Failure.new(name, failed_test, text_content(failure), job, :failure, message_content(failure))
+      failures << Failure.new(name, unit_name, text_content(failure), job, :failure, message_content(failure))
     end
     testcase.elements.each("error") do |error|
-      failures << Failure.new(name, failed_test, text_content(error), job, :error, message_content(error))
+      failures << Failure.new(name, unit_name, text_content(error), job, :error, message_content(error))
     end
   end
 end
 
-STDERR.puts "--- ❓ Checking failures"
+STDERR.puts "--- ✍️ Preparing annotation"
 STDERR.puts "#{testcases} testcases found"
 
-if failures.empty?
-  STDERR.puts "There were no failures/errors 🙌"
-  exit 0
-else
+if failures.any?
   STDERR.puts "There #{failures.length == 1 ? "is 1 failure/error" : "are #{failures.length} failures/errors" } 😭"
-end
 
-STDERR.puts "--- ✍️ Preparing annotation"
+  failures_count = failures.select {|f| f.type == :failure }.length
+  errors_count = failures.select {|f| f.type == :error }.length
+  puts [
+    failures_count == 0 ? nil : (failures_count == 1 ? "1 failure" : "#{failures_count} failures"),
+    errors_count === 0 ? nil : (errors_count == 1 ? "1 error" : "#{errors_count} errors"),
+  ].compact.join(" and ") + ":\n\n"
+
+  failures.each do |failure|
+    puts "<details>"
+    puts "<summary><code>#{failure.name} in #{failure.unit_name}</code></summary>\n\n"
+    if failure.message
+      puts "<p>#{failure.message.chomp.strip}</p>\n\n"
+    end
+    if failure.body
+      puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
+    end
+    if failure.job
+      puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
+    end
+    puts "</details>"
+    puts "" unless failure == failures.last
+  end
 
-failures_count = failures.select {|f| f.type == :failure }.length
-errors_count = failures.select {|f| f.type == :error }.length
+else
+  STDERR.puts "There were no failures/errors 🙌"
+end
 
-puts [
-  failures_count == 0 ? nil : (failures_count == 1 ? "1 failure" : "#{failures_count} failures"),
-  errors_count === 0 ? nil : (errors_count == 1 ? "1 error" : "#{errors_count} errors"),
-].compact.join(" and ") + ":\n\n"
+if report_slowest > 0
+  STDERR.puts "Reporting slowest tests ⏱"
 
-failures.each do |failure|
   puts "<details>"
-  puts "<summary><code>#{failure.name} in #{failure.failed_test}</code></summary>\n\n"
-  if failure.message
-    puts "<p>#{failure.message.chomp.strip}</p>\n\n"
-  end
-  if failure.body
-    puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
-  end
-  if failure.job
-    puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
+  puts "<summary>#{report_slowest} slowest tests</summary>\n\n"
+  puts "<table>"
+  puts "<thead><tr><th>Unit</th><th>Test</th><th>Time</th></tr></thead>"
+  puts "<tbody>"
+  timings.sort_by(&:time).reverse.take(report_slowest).each do |timing|
+    puts "<tr><td>#{timing.unit_name}</td><td>#{timing.name}</td><td>#{timing.time}</td></tr>"
   end
+  puts "</tbody>"
+  puts "</table>"
   puts "</details>"
-  puts "" unless failure == failures.last
 end
+
+exit 64 if failures.any? # special exit code to signal test failures

ruby/tests/annotate_test.rb

@@ -9,7 +9,7 @@
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       8 testcases found
       There were no failures/errors 🙌
     OUTPUT
@@ -24,10 +24,9 @@
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       6 testcases found
       There are 4 failures/errors 😭
-      --- ✍️ Preparing annotation
       4 failures:
       
       <details>
@@ -107,7 +106,7 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "handles failures and errors across multiple files" do
@@ -117,10 +116,9 @@
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       6 testcases found
       There are 4 failures/errors 😭
-      --- ✍️ Preparing annotation
       2 failures and 2 errors:
       
       <details>
@@ -200,18 +198,17 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "accepts custom regex filename patterns for job id" do
     output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=junit-(.*)-custom-pattern.xml", "#{__dir__}/../bin/annotate", "#{__dir__}/custom-job-uuid-pattern/")
 
     assert_equal <<~OUTPUT, output
       Parsing junit-123-456-custom-pattern.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       2 testcases found
       There is 1 failure/error 😭
-      --- ✍️ Preparing annotation
       1 failure:
       
       <details>
@@ -234,7 +231,7 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "uses the file path instead of classname for annotation content when specified" do
@@ -244,10 +241,9 @@
       Parsing junit-1.xml
       Parsing junit-2.xml
       Parsing junit-3.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       6 testcases found
       There are 4 failures/errors 😭
-      --- ✍️ Preparing annotation
       2 failures and 2 errors:
 
       <details>
@@ -327,7 +323,7 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "handles failures across multiple files in sub dirs" do
@@ -337,10 +333,9 @@
       Parsing sub-dir/junit-1.xml
       Parsing sub-dir/junit-2.xml
       Parsing sub-dir/junit-3.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       6 testcases found
       There are 4 failures/errors 😭
-      --- ✍️ Preparing annotation
       4 failures:
       
       <details>
@@ -420,18 +415,17 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "handles empty failure bodies" do
     output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/empty-failure-body/")
 
     assert_equal <<~OUTPUT, output
       Parsing junit.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       2 testcases found
       There is 1 failure/error 😭
-      --- ✍️ Preparing annotation
       1 failure:
 
       <details>
@@ -442,18 +436,17 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "handles missing message attributes" do
     output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/missing-message-attribute/")
 
     assert_equal <<~OUTPUT, output
       Parsing junit.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       4 testcases found
       There are 3 failures/errors 😭
-      --- ✍️ Preparing annotation
       1 failure and 2 errors:
 
       <details>
@@ -472,18 +465,17 @@
       </details>
     OUTPUT
 
-    assert_equal 0, status.exitstatus
+    assert_equal 64, status.exitstatus
   end
 
   it "handles cdata formatted XML files" do
     output, status = Open3.capture2e("#{__dir__}/../bin/annotate", "#{__dir__}/failure-with-cdata/")
 
     assert_equal <<~OUTPUT, output
       Parsing junit.xml
-      --- ❓ Checking failures
+      --- ✍️ Preparing annotation
       2 testcases found
       There is 1 failure/error 😭
-      --- ✍️ Preparing annotation
       1 error:
 
       <details>
@@ -497,6 +489,36 @@
       </details>
     OUTPUT
 
+    assert_equal 64, status.exitstatus
+  end
+
+  it "reports specified amount of slowest tests" do
+    output, status = Open3.capture2e("env", "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=5", "#{__dir__}/../bin/annotate", "#{__dir__}/no-test-failures/")
+
+    assert_equal <<~OUTPUT, output
+      Parsing junit-1.xml
+      Parsing junit-2.xml
+      Parsing junit-3.xml
+      --- ✍️ Preparing annotation
+      8 testcases found
+      There were no failures/errors 🙌
+      Reporting slowest tests ⏱
+      <details>
+      <summary>5 slowest tests</summary>
+
+      <table>
+      <thead><tr><th>Unit</th><th>Test</th><th>Time</th></tr></thead>
+      <tbody>
+      <tr><td>spec.models.account_spec</td><td>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default</td><td>0.977127</td></tr>
+      <tr><td>spec.models.account_spec</td><td>Account#maximum_jobs_added_by_pipeline_changer returns 250 by default</td><td>0.967127</td></tr>
+      <tr><td>spec.models.account_spec</td><td>Account#maximum_jobs_added_by_pipeline_changer returns 500 if the account is ABC</td><td>0.620013</td></tr>
+      <tr><td>spec.models.account_spec</td><td>Account#maximum_jobs_added_by_pipeline_changer returns 900 if the account is F00</td><td>0.520013</td></tr>
+      <tr><td>spec.models.account_spec</td><td>Account#maximum_jobs_added_by_pipeline_changer returns 700 if the account is XYZ</td><td>0.420013</td></tr>
+      </tbody>
+      </table>
+      </details>
+    OUTPUT
+
     assert_equal 0, status.exitstatus
   end
 end