Add Logging and Debugging to Investigate RBE Failures

.github/workflows/bazel.yml

@@ -61,11 +61,26 @@ on:
         required: false
         type: string
         default: ''
-      artifact-name:
+      download-name:
+        description: name of artifact to download
+        required: false
+        type: string
+        default: ''
+      download-path:
+        description: path of artifact to download
+        required: false
+        type: string
+        default: ''
+      upload-name:
         description: Name of artifact to upload
         required: false
         type: string
-        default: 'ignore-artifacts'
+        default: ''
+      upload-path:
+        description: path of artifact to upload
+        required: false
+        type: string
+        default: ''
 
 jobs:
   bazel:
@@ -78,6 +93,7 @@ jobs:
       TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
       TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
       NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      BAZEL_LOG_FILE: bazel-logs/bzl_${{ github.run_id }}.log
       SE_AVOID_STATS: true
     steps:
       - name: Checkout source tree
@@ -155,6 +171,12 @@ jobs:
       - name: Setup curl for Ubuntu
         if: inputs.os == 'ubuntu'
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: "Download artifact"
+        if: ${{ inputs.download-name != '' && inputs.download-path != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.download-name }}
+          path: ${{ inputs.download-path }}
       - name: Run Bazel
         run: ${{ inputs.run }}
       - name: Start SSH session
@@ -171,14 +193,10 @@ jobs:
           title: "Nightly"
           prerelease: true
           files: ${{ inputs.nightly-release-files }}
-      - name: Save changes
-        if: ${{ always() && inputs.artifact-name != 'ignore-artifacts' }}
-        run: |
-          git diff > changes.patch
-      - name: "Upload changes"
-        if: ${{ always() && inputs.artifact-name != 'ignore-artifacts' }}
+      - name: "Upload artifact"
+        if: ${{ always() && inputs.upload-name != '' && inputs.upload-path != '' }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.artifact-name }}
-          path: changes.patch
+          name: ${{ inputs.upload-name }}
+          path: ${{ inputs.upload-path }}
           retention-days: 6

.github/workflows/ci-rbe.yml

@@ -26,4 +26,38 @@ jobs:
       name: All RBE tests
       caching: false
       ruby-version: jruby-9.4.12.0
+      upload-name: bazel-logs
+      upload-path: bazel-logs
       run: ./scripts/github-actions/ci-build.sh
+  retry-remote:
+    name: Retry Failures on RBE
+    needs: test
+    if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: RBE Retries
+      caching: false
+      ruby-version: jruby-9.4.12.0
+      download-name: bazel-logs
+      download-path: bazel-logs
+      upload-name: retry-remote-logs
+      upload-path: bazel-logs
+      run: |
+        ./go retry_failed_tests "${BAZEL_LOG_FILE}" true
+  retry-local:
+    name: Retry Failures on GHA
+    needs: test
+    if: always() && needs.test.result == 'failure' && github.repository_owner == 'seleniumhq' && startsWith(github.head_ref, 'renovate/') != true
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: GHA Retries
+      caching: false
+      ruby-version: jruby-9.4.12.0
+      browser: needs-xvfb
+      java-version: 17
+      download-name: bazel-logs
+      download-path: bazel-logs
+      upload-name: retry-local-logs
+      upload-path: bazel-logs
+      run: |
+        ./go retry_failed_tests "${BAZEL_LOG_FILE}" false

.github/workflows/pin-browsers.yml

@@ -11,8 +11,11 @@ jobs:
     with:
       name: Pin Browsers
       cache-key: pin-browsers
-      run: bazel run //scripts:pinned_browsers
-      artifact-name: pinned-browsers
+      upload-name: pinned-browsers
+      upload-path: changes.patch
+      run: |
+        bazel run //scripts:pinned_browsers
+        git diff > changes.patch
 
   pull-request:
     if: github.repository_owner == 'seleniumhq'

Rakefile

@@ -274,6 +274,45 @@ end
 task test_py: [:py_prep_for_install_release, 'py:marionette_test']
 task build: %i[all firefox remote selenium tests]
 
+desc 'Retry failed tests from a log file'
+task :retry_failed_tests, [:log_file, :rbe] do |_task, arguments|
+  log_file = arguments[:log_file]
+
+  raise 'Error: Please provide a bazel log file containing test results.' if log_file.nil?
+  raise "Error: Log file '#{log_file}' does not exist." unless File.exist?(log_file)
+
+  rbe = arguments[:rbe]
+  failing_tests = []
+
+  File.readlines(log_file).reverse_each do |line|
+    if line.match?(%r{//.+:.*FAILED})
+      failing_tests << line.strip.split[0]
+    elsif !failing_tests.empty? && line.match?(%r{//.+:.*PASSED})
+      break
+    end
+  end
+
+  puts "Found #{failing_tests.size} failing tests; Retrying"
+
+  retry_args = arguments.extras + %w[--test_output=streamed --test_env DEBUG=true]
+  retry_args << (rbe == 'true' ? '--config=remote' : '--pin_browsers')
+
+  retry_failures = false
+  failing_tests.each do |failed_target|
+    target_name = failed_target.split('/').last.tr(':', '_')
+    target_name += rbe ? '_rbe' : '_gha'
+    retry_logs = log_file.sub('.log', "_#{target_name}_retry.log")
+    begin
+      Bazel.execute('test', retry_args, failed_target, verbose: true, log_file: retry_logs)
+    rescue StandardError => e
+      retry_failures = true
+      puts "Failed retry of #{failed_target}: #{e.message}"
+    end
+  end
+
+  raise 'Some tests failed during retry' if retry_failures
+end
+
 desc 'Clean build artifacts.'
 task :clean do
   rm_rf 'build/'

rake_tasks/bazel.rb

@@ -7,9 +7,7 @@
 require_relative 'selenium_rake/checks'
 
 module Bazel
-  def self.execute(kind, args, target, &block)
-    verbose = Rake::FileUtilsExt.verbose_flag
-
+  def self.execute(kind, args, target, verbose: Rake::FileUtilsExt.verbose_flag, log_file: nil, &block)
     if target.end_with?(':run')
       kind = 'run'
       target = target[0, target.length - 4]
@@ -18,28 +16,34 @@ def self.execute(kind, args, target, &block)
     cmd = %w[bazel] + [kind, target] + (args || [])
     cmd_out = ''
     cmd_exit_code = 0
+    puts "Executing:\n#{cmd.join(' ')}" if verbose
 
     if SeleniumRake::Checks.windows?
-      cmd += ['2>&1']
       cmd_line = cmd.join(' ')
-      cmd_out = `#{cmd_line}`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
-      puts cmd_out if verbose
-      cmd_exit_code = $CHILD_STATUS
+      begin
+        cmd_out = `#{cmd_line} 2>&1`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+        puts cmd_out if verbose
+        File.write(log_file, cmd_out) if log_file
+        cmd_exit_code = $CHILD_STATUS.exitstatus
+      rescue => e
+        raise "Windows command execution failed: #{e.message}"
+      end
     else
-      Open3.popen2e(*cmd) do |stdin, stdouts, wait|
-        is_running = true
-        stdin.close
-        while is_running
-          begin
-            stdouts.wait_readable
-            line = stdouts.readpartial(512)
+      begin
+        Open3.popen2e(*cmd) do |stdin, stdouts, wait|
+          stdin.close
+          log = log_file ? File.open(log_file, 'a') : nil
+          while (line = stdouts.gets)
             cmd_out += line
             $stdout.print line if verbose
-          rescue EOFError
-            is_running = false
+            log&.write(line)
+            log&.flush
           end
+          log&.close
+          cmd_exit_code = wait.value.exitstatus
         end
-        cmd_exit_code = wait.value.exitstatus
+      rescue => e
+        raise "Command execution failed: #{e.message}"
       end
     end
 

rb/spec/integration/selenium/webdriver/action_builder_spec.rb

@@ -44,8 +44,8 @@ module WebDriver
           input = driver.find_element(css: '#working')
 
           driver.action.send_keys(input, 'abcd').perform
-          wait.until { input.attribute(:value).length == 4 }
-          expect(input.attribute(:value)).to eq('abcd')
+          wait.until { input.property(:value).length == 4 }
+          expect(input.property(:value)).to eq('abcd')
         end
 
         it 'sends keys with multiple arguments' do
@@ -55,8 +55,8 @@ module WebDriver
           input.click
 
           driver.action.send_keys('abcd', 'dcba').perform
-          wait.until { input.attribute(:value).length == 8 }
-          expect(input.attribute(:value)).to eq('abcddcba')
+          wait.until { input.property(:value).length == 8 }
+          expect(input.property(:value)).to eq('abcddcba')
         end
 
         it 'sends non-ASCII keys' do
@@ -66,8 +66,8 @@ module WebDriver
           input.click
 
           driver.action.send_keys('abcd', :left, 'a').perform
-          wait.until { input.attribute(:value).length == 5 }
-          expect(input.attribute(:value)).to eq('abcad')
+          wait.until { input.property(:value).length == 5 }
+          expect(input.property(:value)).to eq('abcad')
         end
       end
 
@@ -81,9 +81,9 @@ module WebDriver
           event_input.click
 
           driver.action.key_down(:shift).send_keys('ab').key_up(:shift).perform
-          wait.until { event_input.attribute(:value).length == 2 }
+          wait.until { event_input.property(:value).length == 2 }
 
-          expect(event_input.attribute(:value)).to eq('AB')
+          expect(event_input.property(:value)).to eq('AB')
           expected = keylogger.text.strip
           expect(expected).to match(/^(focus )?keydown keydown keypress keyup keydown keypress keyup keyup$/)
         end
@@ -131,10 +131,10 @@ module WebDriver
           event_input = driver.find_element(id: 'clickField')
 
           driver.action.click_and_hold(event_input).perform
-          expect(event_input.attribute(:value)).to eq('Hello')
+          expect(event_input.property(:value)).to eq('Hello')
 
           driver.action.release_actions
-          expect(event_input.attribute(:value)).to eq('Clicked')
+          expect(event_input.property(:value)).to eq('Clicked')
         end
       end
 
@@ -143,24 +143,27 @@ module WebDriver
           driver.navigate.to url_for('javascriptPage.html')
           element = driver.find_element(id: 'clickField')
           driver.action.click(element).perform
-          expect(element.attribute(:value)).to eq('Clicked')
+          expect(element.property(:value)).to eq('Clicked')
         end
 
         it 'executes with equivalent pointer methods' do
           driver.navigate.to url_for('javascriptPage.html')
           element = driver.find_element(id: 'clickField')
           driver.action.move_to(element).pointer_down(:left).pointer_up(:left).perform
-          expect(element.attribute(:value)).to eq('Clicked')
+          expect(element.property(:value)).to eq('Clicked')
         end
       end
 
       describe '#double_click' do
+        # https://issues.chromium.org/issues/400087471
+        before { reset_driver! if GlobalTestEnv.rbe? && GlobalTestEnv.browser == :chrome }
+
         it 'presses pointer twice', except: {browser: %i[safari safari_preview]} do
           driver.navigate.to url_for('javascriptPage.html')
           element = driver.find_element(id: 'doubleClickField')
 
           driver.action.double_click(element).perform
-          expect(element.attribute(:value)).to eq('DoubleClicked')
+          expect(element.property(:value)).to eq('DoubleClicked')
         end
 
         it 'executes with equivalent pointer methods', except: {browser: %i[safari safari_preview]} do
@@ -171,7 +174,7 @@ module WebDriver
                 .pointer_down(:left).pointer_up(:left)
                 .pointer_down(:left).pointer_up(:left)
                 .perform
-          expect(element.attribute(:value)).to eq('DoubleClicked')
+          expect(element.property(:value)).to eq('DoubleClicked')
         end
       end
 
@@ -181,15 +184,15 @@ module WebDriver
           element = driver.find_element(id: 'doubleClickField')
 
           driver.action.context_click(element).perform
-          expect(element.attribute(:value)).to eq('ContextClicked')
+          expect(element.property(:value)).to eq('ContextClicked')
         end
 
         it 'executes with equivalent pointer methods' do
           driver.navigate.to url_for('javascriptPage.html')
           element = driver.find_element(id: 'doubleClickField')
 
           driver.action.move_to(element).pointer_down(:right).pointer_up(:right).perform
-          expect(element.attribute(:value)).to eq('ContextClicked')
+          expect(element.property(:value)).to eq('ContextClicked')
         end
       end
 
@@ -199,7 +202,7 @@ module WebDriver
           element = driver.find_element(id: 'clickField')
           driver.action.move_to(element).click.perform
 
-          expect(element.attribute(:value)).to eq('Clicked')
+          expect(element.property(:value)).to eq('Clicked')
         end
 
         it 'moves to element with offset' do
@@ -212,7 +215,7 @@ module WebDriver
           y_offset = (destination_rect.y - origin_rect.y).ceil
 
           driver.action.move_to(origin, x_offset, y_offset).click.perform
-          expect(destination.attribute(:value)).to eq('Clicked')
+          expect(destination.property(:value)).to eq('Clicked')
         end
       end
 
@@ -256,7 +259,7 @@ module WebDriver
           rect = element.rect
           driver.action.move_to_location(rect.x.ceil, rect.y.ceil).click.perform
 
-          expect(element.attribute(:value)).to eq('Clicked')
+          expect(element.property(:value)).to eq('Clicked')
         end
       end
 

rb/spec/integration/selenium/webdriver/driver_spec.rb

@@ -22,9 +22,11 @@
 module Selenium
   module WebDriver
     describe Driver, exclusive: {bidi: false, reason: 'Not yet implemented with BiDi'} do
+      after { reset_driver! if GlobalTestEnv.rbe? && GlobalTestEnv.browser == :chrome }
+
       it_behaves_like 'driver that can be started concurrently', exclude: [
         {browser: %i[safari safari_preview]},
-        {browser: :firefox, rbe: true, reason: 'https://github.com/mozilla/geckodriver/issues/2219'},
+        {browser: :firefox, reason: 'https://github.com/SeleniumHQ/selenium/issues/15451'},
         {driver: :remote, rbe: true, reason: 'Cannot start 2+ drivers at once.'}
       ]
 
@@ -60,8 +62,7 @@ module WebDriver
 
       it 'refreshes the page' do
         driver.navigate.to url_for('javascriptPage.html')
-        sleep 1 # javascript takes too long to load
-        driver.find_element(id: 'updatediv').click
+        short_wait { driver.find_element(id: 'updatediv') }.click
         expect(driver.find_element(id: 'dynamo').text).to eq('Fish and chips!')
         driver.navigate.refresh
         wait_for_element(id: 'dynamo')