new feature: report specified amount of slowest tests (0 by default)

Author: jvdp

README.md

@@ -56,6 +56,11 @@ Default: `junit`
 
 The buildkite annotation context to use. Useful to differentiate multiple runs of this plugin in a single pipeline.
 
+### `report-slowest` (optional)
+Default: `0`
+
+Adds a table with the specified amount of slowest tests listed in the JUnit XML files. The annotation will also be shown when all tests are successful.
+
 ## Developing
 
 To test the plugin hooks (in Bash) and the junit parser (in Ruby):

hooks/command

@@ -10,6 +10,8 @@ echo "--- :junit: Download the junits"
 artifacts_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-artifacts-tmp.XXXXXXXXXX")"
 annotation_dir="$(pwd)/$(mktemp -d "junit-annotate-plugin-annotation-tmp.XXXXXXXXXX")"
 annotation_path="${annotation_dir}/annotation.md"
+annotation_style="info"
+fail_build=0
 
 function cleanup {
   rm -rf "${artifacts_dir}"
@@ -30,6 +32,7 @@ buildkite-agent artifact download \
 
 echo "--- :junit: Processing the junits"
 
+set +e
 docker \
   --log-level "error" \
   run \
@@ -38,25 +41,35 @@ docker \
     --volume "$PLUGIN_DIR/ruby:/src" \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN:-}" \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT:-}" \
+    --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST:-}" \
     ruby:2.7-alpine ruby /src/bin/annotate /junits \
       > "$annotation_path"
 
+if [[ $? -eq 64 ]]; then # special exit code to signal test failures
+  annotation_style="error"
+  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR:-false}" =~ (true|on|1) ]]; then
+    fail_build=1
+  fi
+fi
+
+set -e
+
 cat "$annotation_path"
 
 if grep -q "<details>" "$annotation_path"; then
-
   if ! check_size; then
-    echo "--- :warning: Failures too large to annotate"
-    echo "The failures are too large to create a build annotation. Please inspect the failed JUnit artifacts manually."
+    echo "--- :warning: Report too large to annotate"
+    echo "The report are too large to create a build annotation. Please inspect the failed JUnit artifacts manually."
   else
     echo "--- :buildkite: Creating annotation"
     # shellcheck disable=SC2002
-    cat "$annotation_path" | buildkite-agent annotate --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT:-junit}" --style error
+    cat "$annotation_path" | buildkite-agent annotate --context "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_CONTEXT:-junit}" --style "$annotation_style"
   fi
+fi
 
-  if [[ "${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAIL_BUILD_ON_ERROR:-false}" =~ (true|on|1) ]]
-  then
-    echo "--- :boom: Failing build due to error"
-    exit 1
-  fi
+if ((fail_build)); then
+  echo "--- :boom: Failing build due to error"
+  exit 1
+else
+  exit 0
 fi

plugin.yml

@@ -18,6 +18,8 @@ configuration:
       type: boolean
     context:
       type: string
+    report-slowest:
+      type: integer
   required:
     - artifacts
   additionalProperties: false

ruby/bin/annotate

@@ -17,12 +17,18 @@ job_pattern = '-(.*).xml' if !job_pattern || job_pattern.empty?
 failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT']
 failure_format = 'classname' if !failure_format || failure_format.empty?
 
-class Failure < Struct.new(:name, :failed_test, :body, :job, :type, :message)
+report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST'].to_i
+
+class Failure < Struct.new(:name, :unit_name, :body, :job, :type, :message)
+end
+
+class Timing < Struct.new(:name, :unit_name, :time)
 end
 
 junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"))
 testcases = 0
 failures = []
+timings = []
 
 def text_content(element)
   # Handle mulptiple CDATA/text children elements
@@ -55,48 +61,65 @@ junit_report_files.sort.each do |file|
   REXML::XPath.each(doc, '//testsuite/testcase') do |testcase|
     testcases += 1
     name = testcase.attributes['name'].to_s
-    failed_test = testcase.attributes[failure_format].to_s
+    unit_name = testcase.attributes[failure_format].to_s
+    time = testcase.attributes['time'].to_f
+    timings << Timing.new(name, unit_name, time)
     testcase.elements.each("failure") do |failure|
-      failures << Failure.new(name, failed_test, text_content(failure), job, :failure, message_content(failure))
+      failures << Failure.new(name, unit_name, text_content(failure), job, :failure, message_content(failure))
     end
     testcase.elements.each("error") do |error|
-      failures << Failure.new(name, failed_test, text_content(error), job, :error, message_content(error))
+      failures << Failure.new(name, unit_name, text_content(error), job, :error, message_content(error))
     end
   end
 end
 
-STDERR.puts "--- ❓ Checking failures"
+STDERR.puts "--- ✍️ Preparing annotation"
 STDERR.puts "#{testcases} testcases found"
 
-if failures.empty?
-  STDERR.puts "There were no failures/errors 🙌"
-  exit 0
-else
+if failures.any?
   STDERR.puts "There #{failures.length == 1 ? "is 1 failure/error" : "are #{failures.length} failures/errors" } 😭"
-end
 
-STDERR.puts "--- ✍️ Preparing annotation"
+  failures_count = failures.select {|f| f.type == :failure }.length
+  errors_count = failures.select {|f| f.type == :error }.length
+  puts [
+    failures_count == 0 ? nil : (failures_count == 1 ? "1 failure" : "#{failures_count} failures"),
+    errors_count === 0 ? nil : (errors_count == 1 ? "1 error" : "#{errors_count} errors"),
+  ].compact.join(" and ") + ":\n\n"
+
+  failures.each do |failure|
+    puts "<details>"
+    puts "<summary><code>#{failure.name} in #{failure.unit_name}</code></summary>\n\n"
+    if failure.message
+      puts "<p>#{failure.message.chomp.strip}</p>\n\n"
+    end
+    if failure.body
+      puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
+    end
+    if failure.job
+      puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
+    end
+    puts "</details>"
+    puts "" unless failure == failures.last
+  end
 
-failures_count = failures.select {|f| f.type == :failure }.length
-errors_count = failures.select {|f| f.type == :error }.length
+else
+  STDERR.puts "There were no failures/errors 🙌"
+end
 
-puts [
-  failures_count == 0 ? nil : (failures_count == 1 ? "1 failure" : "#{failures_count} failures"),
-  errors_count === 0 ? nil : (errors_count == 1 ? "1 error" : "#{errors_count} errors"),
-].compact.join(" and ") + ":\n\n"
+if report_slowest > 0
+  STDERR.puts "Reporting slowest tests ⏱"
 
-failures.each do |failure|
   puts "<details>"
-  puts "<summary><code>#{failure.name} in #{failure.failed_test}</code></summary>\n\n"
-  if failure.message
-    puts "<p>#{failure.message.chomp.strip}</p>\n\n"
-  end
-  if failure.body
-    puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
-  end
-  if failure.job
-    puts "in <a href=\"##{failure.job}\">Job ##{failure.job}</a>"
+  puts "<summary>#{report_slowest} slowest tests</summary>\n\n"
+  puts "<table>"
+  puts "<thead><tr><th>Unit</th><th>Test</th><th>Time</th></tr></thead>"
+  puts "<tbody>"
+  timings.sort_by(&:time).reverse.take(report_slowest).each do |timing|
+    puts "<tr><td>#{timing.unit_name}</td><td>#{timing.name}</td><td>#{timing.time}</td></tr>"
   end
+  puts "</tbody>"
+  puts "</table>"
   puts "</details>"
-  puts "" unless failure == failures.last
 end
+
+exit 64 if failures.any? # special exit code to signal test failures