Merge pull request #223 from buildkite-plugins/toote_skipped_and_escaping

Skipped tests and output escaping

Author: pzeballos

README.md

@@ -40,7 +40,7 @@ The buildkite annotation context to use. Useful to differentiate multiple runs o
 
 Default: `-(.*).xml`
 
-The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation. 
+The regular expression (with capture group) that matches the job UUID in the junit file names. This is used to create the job links in the annotation.
 
 To use this, configure your test reporter to embed the `$BUILDKITE_JOB_ID` environment variable into your junit file names. For example `"junit-buildkite-job-$BUILDKITE_JOB_ID.xml"`.
 
@@ -54,7 +54,7 @@ There are two options for this:
 * `file`
   * displays: `MyClass::UnderTest text of the failed expectation in path/to/my_class/under_test.file_ext`
 
-### `fail-build-on-error` (optional)  
+### `fail-build-on-error` (optional)
 
 Default: `false`
 
@@ -70,6 +70,12 @@ Exit code of the plugin if the call to `buildkite-agent artifact download` fails
 
 Minimum amount of run tests that need to be analyzed or a failure will be reported. It is useful to ensure that tests are actually run and report files to analyze do contain information.
 
+### `report-skipped` (optional, boolean)
+
+Default: `false`
+
+Will add a list of skipped tests at the end of the annotation. Note that even if there are skipped tests, the annotation may not be added unless other options or results of the processing forces it to.
+
 ### `report-slowest` (optional)
 
 Default: `0`

hooks/command

@@ -51,6 +51,7 @@ docker \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_JOB_UUID_FILE_PATTERN:-}" \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT:-}" \
     --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST:-}" \
+    --env "BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED=${BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED:-}" \
     "${RUBY_IMAGE}" ruby /src/bin/annotate /junits \
       > "$annotation_path"
 
@@ -78,9 +79,9 @@ if [ $has_errors -eq 0 ]; then
     echo "Will create annotation anyways"
     create_annotation=1
   fi
-  
+
   if [[ -e "${annotation_path}" ]]; then
-    TOTAL_TESTS=$(head -4 "${annotation_path}" | grep 'Total tests' | cut -d\  -f3)
+    TOTAL_TESTS=$(head -5 "${annotation_path}" | grep 'Total tests' | cut -d\  -f3)
   else
     TOTAL_TESTS=0
   fi
@@ -95,7 +96,7 @@ elif ! check_size; then
 
   # creating a simplified version of the annotation
   mv "${annotation_path}" "${annotation_path}2"
-  head -4 "${annotation_path}2" >"${annotation_path}"
+  head -5 "${annotation_path}2" >"${annotation_path}"
   # || true is to avoid issues if no summary is found
   grep '<summary>' "${annotation_path}2" >>"${annotation_path}" || true
 

plugin.yml

@@ -24,6 +24,8 @@ configuration:
       type: string
     min-tests:
       type: integer
+    report-skipped:
+      type: boolean
     report-slowest:
       type: integer
     ruby-image:

ruby/bin/annotate

@@ -18,16 +18,21 @@ failure_format = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_FAILURE_FORMAT']
 failure_format = 'classname' if !failure_format || failure_format.empty?
 
 report_slowest = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SLOWEST'].to_i
+report_skipped = ENV['BUILDKITE_PLUGIN_JUNIT_ANNOTATE_REPORT_SKIPPED'] == 'true'
 
-class Failure < Struct.new(:name, :unit_name, :body, :job, :type, :message)
+class Failure < Struct.new(:name, :unit_name, :body, :job, :message)
 end
 
 class Timing < Struct.new(:name, :unit_name, :time)
 end
 
 junit_report_files = Dir.glob(File.join(junits_dir, "**", "*"), File::FNM_DOTMATCH)
 testcases = 0
-failures = []
+tests = {
+  failure: [],
+  error: [],
+  skipped: []
+}
 timings = []
 
 def text_content(element)
@@ -64,30 +69,27 @@ junit_report_files.sort.each do |file|
     unit_name = testcase.attributes[failure_format].to_s
     time = testcase.attributes['time'].to_f
     timings << Timing.new(name, unit_name, time)
-    testcase.elements.each("failure") do |failure|
-      failures << Failure.new(name, unit_name, text_content(failure), job, :failure, message_content(failure))
-    end
-    testcase.elements.each("error") do |error|
-      failures << Failure.new(name, unit_name, text_content(error), job, :error, message_content(error))
+    testcase.elements.each("failure | error | skipped") do |elem|
+      tests[elem.name.to_sym] << Failure.new(name, unit_name, text_content(elem), job, message_content(elem))
     end
   end
 end
 
 STDERR.puts "--- ✍️ Preparing annotation"
 
-failures_count = failures.select {|f| f.type == :failure }.length
-errors_count = failures.select {|f| f.type == :error }.length
-
-puts "Failures: #{failures_count}"
-puts "Errors: #{errors_count}"
+puts "Failures: #{tests[:failure].length}"
+puts "Errors: #{tests[:error].length}"
+puts "Skipped: #{tests[:skipped].length}"
 puts "Total tests: #{testcases}"
 
-failures.each do |failure|
+skipped = tests.delete(:skipped) # save value for later
+
+tests.values.flatten.each do |failure|
   puts ""
   puts "<details>"
-  puts "<summary><code>#{failure.name} in #{failure.unit_name}</code></summary>\n\n"
+  puts "<summary><code>#{CGI.escapeHTML failure.name} in #{CGI.escapeHTML failure.unit_name}</code></summary>\n\n"
   if failure.message
-    puts "<p>#{failure.message.chomp.strip}</p>\n\n"
+    puts "<p>#{CGI.escapeHTML failure.message.chomp.strip}</p>\n\n"
   end
   if failure.body
     puts "<pre><code>#{CGI.escapeHTML(failure.body.chomp.strip)}</code></pre>\n\n"
@@ -114,4 +116,17 @@ if report_slowest > 0
   puts "</details>"
 end
 
-exit 64 if failures.any? # special exit code to signal test failures
+if report_skipped
+  STDERR.puts "Reporting skipped tests"
+  puts ""
+  puts "<details>"
+  puts "<summary>#{skipped.length} tests skipped</summary>\n\n"
+  puts "<ol>"
+  skipped.each do |sk|
+    puts "<li>#{CGI.escapeHTML sk.name} in #{CGI.escapeHTML sk.unit_name} (#{CGI.escapeHTML sk.message || "no reason"})</li>\n"
+  end
+  puts "</ol>"
+  puts "</details>"
+end
+
+exit 64 if tests.values.flatten.any? # special exit code to signal test failures