[build] implement steps for running failing rbe tests with debugging turned on

Author: titusfortner

.github/workflows/ci-rbe.yml

@@ -7,6 +7,9 @@ on:
       - trunk
   workflow_dispatch:
 
+env:
+  BAZEL_LOG_FILE: bazel-logs/bazel_${{ github.workflow }}_${{ github.run_id }}.log
+
 jobs:
   format:
     name: Format
@@ -26,4 +29,36 @@ jobs:
       name: All RBE tests
       caching: false
       ruby-version: jruby-9.4.12.0
+      upload-name: bazel-logs
+      upload-path: ${{ env.BAZEL_LOG_FILE }}
       run: ./scripts/github-actions/ci-build.sh
+  retry-remote:
+    name: Retry Failed Tests on RBE
+    needs: test
+    if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: Retry failed tests
+      caching: false
+      ruby-version: jruby-9.4.12.0
+      download-name: bazel-logs
+      download-path: bazel-logs
+      upload-name: remote-retry-logs
+      upload-path: bazel-logs
+      run: |
+        ./go retry_failed_tests "${BAZEL_LOG_FILE}" true
+  retry-local:
+    name: Retry Failed Tests on Local
+    needs: test
+    if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: Retry failed tests
+      caching: false
+      ruby-version: jruby-9.4.12.0
+      download-name: bazel-logs
+      download-path: bazel-logs
+      upload-name: remote-local-logs
+      upload-path: bazel-logs
+      run: |
+        ./go retry_failed_tests "${BAZEL_LOG_FILE}" false

Rakefile

@@ -249,54 +249,68 @@ task test_py: [:py_prep_for_install_release, 'py:marionette_test']
 task build: %i[all firefox remote selenium tests]
 
 desc 'Run on RBE'
-task :run_remote do |_task, arguments|
+task :run_remote [:target] do |_task, arguments|
+  original_ruby_version = File.read('rb/.ruby-version').strip
   File.write('rb/.ruby-version', 'jruby-9.4.12.0')
 
   Bazel.verbose = true
-  log_file = 'bazel_20250314_110017.log'
-  #  log_file = "bazel_#{Time.now.strftime('%Y%m%d_%H%M%S')}.log"
+  log_file = "bazel_#{Time.now.strftime('%Y%m%d_%H%M%S')}.log"
   Bazel.log_file = log_file
 
-  args = arguments.to_a.compact
-  args.shift
-  target = args.shift
-  remote_args = ['--config=remote']
+  target = arguments[:target]
+  remote_args = arguments.extras + ['--config=remote']
 
   has_filters = args.any? { |arg| arg.start_with?('--test_tag_filters=') }
   remote_args += args.map { |arg| arg.start_with?('--test_tag_filters=') ? "#{arg},-skip-remote,-remote" : arg }
   remote_args << '--test_tag_filters=-skip-remote,-remote' unless has_filters
 
   begin
-    # Toggle which line here is commented out for using existing logs or not
-    # Bazel.execute(command, remote_args, target)
-    raise 'Error to run previous failures'
-  rescue StandardError => e
-    failing_tests = []
-
-    File.readlines(log_file).reverse_each do |line|
-      if line.match?(%r{//.+:.*FAILED})
-        failing_tests << line.strip.split(' ')[0]
-      elsif !failing_tests.empty? && line.match?(%r{//.+:.*PASSED})
-        break
-      else
-        next
-      end
+    Bazel.execute(command, remote_args, target)
+  rescue StandardError => _e
+    Rake::Task['retry_failed_tests'].invoke(log_file, true)
+  ensure
+    File.write('rb/.ruby-version', original_ruby_version)
+  end
+end
+
+desc 'Retry failed tests from a log file'
+task :retry_failed_tests, [:log_file, :remote] do |_task, arguments|
+  log_file = arguments[:log_file]
+  raise 'No log file specified' unless log_file
+  raise 'Log file does not exist' unless File.exist?(log_file)
+  remote = arguments[:remote]
+  Bazel.verbose = true
+
+  failing_tests = []
+
+  File.readlines(log_file).reverse_each do |line|
+    if line.match?(%r{//.+:.*FAILED})
+      failing_tests << line.strip.split(' ')[0]
+    elsif !failing_tests.empty? && line.match?(%r{//.+:.*PASSED})
+      break
     end
+  end
 
-    puts "Found #{failing_tests.size} failing tests; Retrying"
+  puts "Found #{failing_tests.size} failing tests; Retrying"
 
-    retry_args = %w[--test_output=streamed --test_env DEBUG=true]
-    # retry_args += remote_args
+  retry_args = arguments.extras + %w[--test_output=streamed --test_env DEBUG=true]
+  retry_args << '--config=remote' if remote
 
-    failing_tests.each do |failed_target|
-      target_name = failed_target.split('/').last.tr(':', '_')
-      retry_logs = log_file.sub('.log', "#{target_name}_local_retry.log")
-      Bazel.log_file = retry_logs
+  retry_failures = false
+  failing_tests.each do |failed_target|
+    target_name = failed_target.split('/').last.tr(':', '_')
+    target_name += remote ? '_remote' : '_local'
+    retry_logs = log_file.sub('.log', "#{target_name}_retry.log")
+    Bazel.log_file = retry_logs
+    begin
       Bazel.execute('test', retry_args, failed_target)
     rescue StandardError => e
-      puts "Failed retry of #{target}: #{e.message}"
+      retry_failures = true
+      puts "Failed retry of #{failed_target}: #{e.message}"
     end
   end
+
+  raise 'Some tests failed during retry' if retry_failures
 end
 
 desc 'Clean build artifacts.'
@@ -342,7 +356,7 @@ task javadocs: %i[//java/src/org/openqa/selenium/grid:all-javadocs] do
       }
 
     STYLE
-              )
+    )
   end
 end
 
@@ -1205,7 +1219,7 @@ def updated_version(current, desired = nil, nightly = nil)
 end
 
 def update_gh_pages
-  @git.fetch('origin', {ref: 'gh-pages'})
+  @git.fetch('origin', { ref: 'gh-pages' })
   @git.checkout('gh-pages', force: true)
 
   %w[java rb py dotnet javascript].each do |language|

rake_tasks/bazel.rb

@@ -7,49 +7,73 @@
 require_relative 'selenium_rake/checks'
 
 module Bazel
-  def self.execute(kind, args, target, &block)
-    verbose = Rake::FileUtilsExt.verbose_flag
+  class << self
+    attr_accessor :log_file, :verbose
 
-    if target.end_with?(':run')
-      kind = 'run'
-      target = target[0, target.length - 4]
+    def verbose?
+      defined?(verbose) ? verbose : Rake::FileUtilsExt.verbose_flag
     end
 
-    cmd = %w[bazel] + [kind, target] + (args || [])
-    cmd_out = ''
-    cmd_exit_code = 0
-
-    if SeleniumRake::Checks.windows?
-      cmd += ['2>&1']
-      cmd_line = cmd.join(' ')
-      cmd_out = `#{cmd_line}`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
-      puts cmd_out if verbose
-      cmd_exit_code = $CHILD_STATUS
-    else
-      Open3.popen2e(*cmd) do |stdin, stdouts, wait|
-        is_running = true
-        stdin.close
-        while is_running
-          begin
-            stdouts.wait_readable
-            line = stdouts.readpartial(512)
+    def execute(kind, args, target, &block)
+      if target.end_with?(':run')
+        kind = 'run'
+        target = target[0, target.length - 4]
+      end
+
+      cmd = %w[bazel] + [kind, target] + (args || [])
+      cmd_out = ''
+      cmd_exit_code = 0
+
+      puts "Logging to: #{log_file}" if verbose? && log_file
+
+      if SeleniumRake::Checks.windows?
+        cmd += ['2>&1']
+        cmd_line = cmd.join(' ')
+        cmd_out = `#{cmd_line}`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+        puts cmd_out if verbose?
+        File.write(log_file, cmd_out) if log_file
+        cmd_exit_code = $CHILD_STATUS
+      else
+        puts "Executing:\n#{cmd.join(' ')}" if verbose?
+        Open3.popen2e(*cmd) do |stdin, stdouts, wait|
+          stdin.close
+          log = log_file ? File.open(log_file, 'a') : nil
+
+          process_output(stdouts) do |line|
             cmd_out += line
-            $stdout.print line if verbose
-          rescue EOFError
-            is_running = false
+            $stdout.print line if verbose?
+            log&.write(line)
+            log&.flush
           end
+
+          log&.close
+          cmd_exit_code = wait.value.exitstatus
         end
-        cmd_exit_code = wait.value.exitstatus
       end
-    end
 
-    raise "#{cmd.join(' ')} failed with exit code: #{cmd_exit_code}\nOutput: #{cmd_out}" if cmd_exit_code != 0
+      raise "#{cmd.join(' ')} failed with exit code: #{cmd_exit_code}\nOutput: #{cmd_out}" if cmd_exit_code != 0
 
-    block&.call(cmd_out)
-    return unless cmd_out =~ %r{\s+(bazel-bin/\S+)}
+      block&.call(cmd_out)
+      return unless cmd_out =~ %r{\s+(bazel-bin/\S+)}
 
-    out_artifact = Regexp.last_match(1)
-    puts "#{target} -> #{out_artifact}" if out_artifact
-    out_artifact
+      out_artifact = Regexp.last_match(1)
+      puts "#{target} -> #{out_artifact}" if out_artifact
+      out_artifact
+    end
+
+    private
+
+    def process_output(stdouts)
+      is_running = true
+      while is_running
+        begin
+          stdouts.wait_readable
+          line = stdouts.readpartial(512)
+          yield line
+        rescue EOFError
+          is_running = false
+        end
+      end
+    end
   end
 end

scripts/github-actions/ci-build.sh

@@ -4,12 +4,16 @@ set -eufo pipefail
 # We want to see what's going on
 set -x
 
+# Create the log directory if it doesn't exist
+LOG_DIR=$(dirname "$BAZEL_LOG_FILE")
+mkdir -p "$LOG_DIR"
+
 # Now run the tests. The engflow build uses pinned browsers
 # so this should be fine
 # shellcheck disable=SC2046
 bazel test --config=remote-ci --build_tests_only \
   --keep_going --flaky_test_attempts=2 \
-  //... -- $(cat .skipped-tests | tr '\n' ' ')
+  //... -- $(cat .skipped-tests | tr '\n' ' ')  2>&1 | tee "$BAZEL_LOG_FILE"
 
 # Build the packages we want to ship to users
 bazel build --config=remote-ci --build_tag_filters=release-artifact //...