Collect test results and print profile summary at the end

Author: zzak

railties/Rakefile

@@ -66,12 +66,13 @@ namespace :test do
 
     # Only generate the template app once.
     require_relative "test/isolation/abstract_unit"
+    require "minitest/rails_plugin"
 
     failing_files = []
 
     dirs = (ENV["TEST_DIR"] || ENV["TEST_DIRS"] || "**").split(",")
     test_options = ENV["TESTOPTS"].to_s.split(/[\s]+/)
-    test_options << "--profile" if ENV["CI"]
+    test_options << "--profile" if ENV["BUILDKITE"]
 
     test_patterns = dirs.map { |dir| "test/#{dir}/*_test.rb" }
     test_files = Dir[*test_patterns].select do |file|
@@ -95,6 +96,8 @@ namespace :test do
       test_files = buckets[n]
     end
 
+    output_file = Tempfile.new("railties_test_reporter")
+
     test_files.each do |file|
       puts "--- #{file}"
       fake_command = Shellwords.join([
@@ -109,6 +112,7 @@ namespace :test do
         # We could run these in parallel, but pretty much all of the
         # railties tests already run in parallel, so ¯\_(⊙︿⊙)_/¯
         Process.waitpid fork {
+          ENV["RAILTIES_OUTPUT_FILE"] = output_file.path
           ARGV.clear.concat test_options
           Rake.application = nil
 
@@ -124,9 +128,22 @@ namespace :test do
       end
     end
 
-    puts "--- All tests completed"
+    puts "+++ All tests completed"
+
+    if ENV["BUILDKITE"]
+      TestResult = Struct.new(:NAME, :failures, :assertions, :klass, :time, :source_location, :location)
+      profile = Minitest::ProfileReporter.new($stdout, profile: 10)
+      output_file.rewind
+      output_file.each_line do |result|
+        data = JSON.parse(result)
+        profile.results << TestResult.new(*data.values)
+      end
+
+      profile.summary
+    end
+
     unless failing_files.empty?
-      puts "^^^ +++"
+      puts "+++"
       puts
       puts "Failed in:"
       failing_files.each do |file|
@@ -136,6 +153,9 @@ namespace :test do
 
       exit 1
     end
+  ensure
+    output_file&.close
+    output_file&.unlink
   end
 end
 
@@ -144,6 +164,6 @@ Rake::TestTask.new("test:regular") do |t|
   t.pattern = "test/**/*_test.rb"
   t.warning = true
   t.verbose = true
-  t.options = "--profile" if ENV["CI"]
+  t.options = "--profile" if ENV["BUILDKITE"]
   t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
 end

railties/lib/minitest/rails_plugin.rb

@@ -26,37 +26,58 @@ def aggregated_results(*)
   end
 
   class ProfileReporter < Reporter
+    attr_accessor :results
+
     def initialize(io = $stdout, options = {})
       super
       @results = []
       @count = options[:profile]
     end
 
     def record(result)
-      @results << result
+      if output_file = ENV["RAILTIES_OUTPUT_FILE"]
+        File.open(output_file, "a") do |f|
+          # Round-trip for re-serialization
+          data = JSON.parse(result.to_json)
+          data[:location] = result.location
+          f.puts(data.to_json)
+        end
+      else
+        @results << result
+      end
     end
 
     def passed?
       true
     end
 
     def report
-      total_time = @results.sum(&:time)
+      # Skip if we're outputting to a file
+      return if ENV["RAILTIES_OUTPUT_FILE"]
+      print_summary
+    end
+
+    def summary
+      print_summary
+    end
+
+    private
+      def print_summary
+        total_time = @results.sum(&:time)
 
-      @results.sort! { |a, b| b.time <=> a.time }
-      slow_results = @results.take(@count)
-      slow_tests_total_time = slow_results.sum(&:time)
+        @results.sort! { |a, b| b.time <=> a.time }
+        slow_results = @results.take(@count)
+        slow_tests_total_time = slow_results.sum(&:time)
 
-      ratio = (total_time == 0) ? 0.0 : (slow_tests_total_time / total_time) * 100
+        ratio = (total_time == 0) ? 0.0 : (slow_tests_total_time / total_time) * 100
 
-      io.puts("\nTop %d slowest tests (%.2f seconds, %.1f%% of total time):\n" % [slow_results.size, slow_tests_total_time, ratio])
-      slow_results.each do |result|
-        io.puts("  %s\n    %.4f seconds %s\n" % [result.location, result.time, source_location(result)])
+        io.puts("\nTop %d slowest tests (%.2f seconds, %.1f%% of total time):\n" % [slow_results.size, slow_tests_total_time, ratio])
+        slow_results.each do |result|
+          io.puts("  %s\n    %.4f seconds %s\n" % [result.location, result.time, source_location(result)])
+        end
+        io.puts("\n")
       end
-      io.puts("\n")
-    end
 
-    private
       def source_location(result)
         filename, line = result.source_location
         return "" unless filename