Merge branch 'main' into feature/clear-ar-connections-flag

Author: rafael-pissardo

.github/workflows/main.yml

@@ -20,6 +20,7 @@ jobs:
   tests:
     name: Tests
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
@@ -29,10 +30,12 @@ jobs:
         - 3.3
         - 3.4
         database: [ mysql, postgres, sqlite ]
-        gemfile: [ rails_7_1, rails_7_2, rails_8_0, rails_main ]
+        gemfile: [ rails_7_1, rails_7_2, rails_8_0, rails_8_1, rails_main ]
         exclude:
           - ruby-version: "3.1"
             gemfile: rails_8_0
+          - ruby-version: "3.1"
+            gemfile: rails_8_1
           - ruby-version: "3.1"
             gemfile: rails_main
     services:
@@ -41,7 +44,7 @@ jobs:
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
         ports:
-          - 33060:3306
+          - 33066:3306
         options: --health-cmd "mysql -h localhost -e \"select now()\"" --health-interval 1s --health-timeout 5s --health-retries 30
       postgres:
         image: postgres:15.1
@@ -52,6 +55,7 @@ jobs:
     env:
       TARGET_DB: ${{ matrix.database }}
       BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
+      RAILS_ENV: test
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -68,3 +72,12 @@ jobs:
           bin/rails db:setup
       - name: Run tests
         run: bin/rails test
+      - name: Upload logs on failure
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ matrix.database }}-${{ matrix.gemfile }}-ruby${{ matrix.ruby-version }}-attempt${{ github.run_attempt }}
+          path: |
+            test/dummy/log/test.log
+          if-no-files-found: ignore
+          retention-days: 30

Appraisals

@@ -15,6 +15,10 @@ appraise "rails-8-0" do
   gem "railties", "~> 8.0.0"
 end
 
+appraise "rails-8-1" do
+  gem "railties", "~> 8.1.0"
+end
+
 appraise "rails-main" do
   gem "railties", github: "rails/rails", branch: "main"
 end

Gemfile.lock

@@ -1,43 +1,43 @@
 PATH
   remote: .
   specs:
-    solid_queue (1.2.1)
+    solid_queue (1.2.4)
       activejob (>= 7.1)
       activerecord (>= 7.1)
       concurrent-ruby (>= 1.3.1)
-      fugit (~> 1.11.0)
+      fugit (~> 1.11)
       railties (>= 7.1)
       thor (>= 1.3.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (7.1.5.1)
-      actionview (= 7.1.5.1)
-      activesupport (= 7.1.5.1)
+    actionpack (7.1.5.2)
+      actionview (= 7.1.5.2)
+      activesupport (= 7.1.5.2)
       nokogiri (>= 1.8.5)
       racc
       rack (>= 2.2.4)
       rack-session (>= 1.0.1)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    actionview (7.1.5.1)
-      activesupport (= 7.1.5.1)
+    actionview (7.1.5.2)
+      activesupport (= 7.1.5.2)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
-    activejob (7.1.5.1)
-      activesupport (= 7.1.5.1)
+    activejob (7.1.5.2)
+      activesupport (= 7.1.5.2)
       globalid (>= 0.3.6)
-    activemodel (7.1.5.1)
-      activesupport (= 7.1.5.1)
-    activerecord (7.1.5.1)
-      activemodel (= 7.1.5.1)
-      activesupport (= 7.1.5.1)
+    activemodel (7.1.5.2)
+      activesupport (= 7.1.5.2)
+    activerecord (7.1.5.2)
+      activemodel (= 7.1.5.2)
+      activesupport (= 7.1.5.2)
       timeout (>= 0.4.0)
-    activesupport (7.1.5.1)
+    activesupport (7.1.5.2)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -55,18 +55,18 @@ GEM
       rake
       thor (>= 0.14.0)
     ast (2.4.2)
-    base64 (0.2.0)
-    benchmark (0.4.0)
-    bigdecimal (3.1.9)
+    base64 (0.3.0)
+    benchmark (0.4.1)
+    bigdecimal (3.3.1)
     builder (3.3.0)
-    concurrent-ruby (1.3.4)
-    connection_pool (2.4.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.4)
     crass (1.0.6)
     date (3.4.1)
     debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    drb (2.2.1)
+    drb (2.2.3)
     erubi (1.13.1)
     et-orbi (1.2.11)
       tzinfo
@@ -75,7 +75,7 @@ GEM
       raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    i18n (1.14.6)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     io-console (0.8.0)
     irb (1.14.3)
@@ -87,19 +87,19 @@ GEM
     loofah (2.23.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
-    minitest (5.25.4)
+    minitest (5.26.0)
     mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
     mutex_m (0.3.0)
     mysql2 (0.5.6)
     nio4r (2.7.4)
-    nokogiri (1.18.0-aarch64-linux-gnu)
+    nokogiri (1.18.9-aarch64-linux-gnu)
       racc (~> 1.4)
-    nokogiri (1.18.0-arm64-darwin)
+    nokogiri (1.18.9-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.0-x86_64-darwin)
+    nokogiri (1.18.9-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.18.0-x86_64-linux-gnu)
+    nokogiri (1.18.9-x86_64-linux-gnu)
       racc (~> 1.4)
     parallel (1.26.3)
     parser (3.3.6.0)
@@ -109,12 +109,13 @@ GEM
     psych (5.2.2)
       date
       stringio
-    puma (6.4.3)
+    puma (7.0.2)
       nio4r (~> 2.0)
     raabro (1.4.0)
     racc (1.8.1)
-    rack (3.1.8)
-    rack-session (2.0.0)
+    rack (3.2.3)
+    rack-session (2.1.1)
+      base64 (>= 0.1.0)
       rack (>= 3.0.0)
     rack-test (2.2.0)
       rack (>= 1.3)
@@ -127,9 +128,9 @@ GEM
     rails-html-sanitizer (1.6.2)
       loofah (~> 2.21)
       nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
-    railties (7.1.5.1)
-      actionpack (= 7.1.5.1)
-      activesupport (= 7.1.5.1)
+    railties (7.1.5.2)
+      actionpack (= 7.1.5.2)
+      activesupport (= 7.1.5.2)
       irb
       rackup (>= 1.0.0)
       rake (>= 12.2)
@@ -203,7 +204,7 @@ DEPENDENCIES
   mocha
   mysql2
   pg
-  puma
+  puma (~> 7.0)
   rdoc
   rubocop-rails-omakase
   solid_queue!

README.md

@@ -342,7 +342,7 @@ When receiving a `QUIT` signal, if workers still have jobs in-flight, these will
 
 If processes have no chance of cleaning up before exiting (e.g. if someone pulls a cable somewhere), in-flight jobs might remain claimed by the processes executing them. Processes send heartbeats, and the supervisor checks and prunes processes with expired heartbeats. Jobs that were claimed by processes with an expired heartbeat will be marked as failed with a `SolidQueue::Processes::ProcessPrunedError`. You can configure both the frequency of heartbeats and the threshold to consider a process dead. See the section below for this.
 
-In a similar way, if a worker is terminated in any other way not initiated by the above signals (e.g. a worker is sent a `KILL` signal), jobs in progress will be marked as failed so that they can be inspected, with a `SolidQueue::Processes::Process::ProcessExitError`. Sometimes a job in particular is responsible for this, for example, if it has a memory leak and you have a mechanism to kill processes over a certain memory threshold, so this will help identifying this kind of situation.
+In a similar way, if a worker is terminated in any other way not initiated by the above signals (e.g. a worker is sent a `KILL` signal), jobs in progress will be marked as failed so that they can be inspected, with a `SolidQueue::Processes::ProcessExitError`. Sometimes a job in particular is responsible for this, for example, if it has a memory leak and you have a mechanism to kill processes over a certain memory threshold, so this will help identifying this kind of situation.
 
 
 ### Database configuration
@@ -519,11 +519,11 @@ DeliverAnnouncementToContactJob.set(wait: 30.minutes).perform_later(contact)
 
 The 3 jobs will go into the scheduled queue and will wait there until they're due. Then, 10 minutes after, the first two jobs will be enqueued and the second one most likely will be blocked because the first one will be running first. Then, assuming the jobs are fast and finish in a few seconds, when the third job is due, it'll be enqueued normally.
 
-Normally scheduled jobs are enqueued in batches, but with concurrency controls, jobs need to be enqueued one by one. This has an impact on performance, similarly to the impact of concurrency controls in bulk enqueuing. Read below for more details. I'd generally advise against mixing concurrency controls with waiting/scheduling in the future.
+Normally scheduled jobs are enqueued in batches, but with concurrency controls, jobs need to be enqueued one by one. This has an impact on performance, similarly to the impact of concurrency controls in bulk enqueuing. Read below for more details. We generally advise against mixing concurrency controls with waiting/scheduling in the future.
 
 ### Performance considerations
 
-Concurrency controls introduce significant overhead (blocked executions need to be created and promoted to ready, semaphores need to be created and updated) so you should consider carefully whether you need them. For throttling purposes, where you plan to have `limit` significantly larger than 1, I'd encourage relying on a limited number of workers per queue instead. For example:
+Concurrency controls introduce significant overhead (blocked executions need to be created and promoted to ready, semaphores need to be created and updated) so you should consider carefully whether you need them. For throttling purposes, where you plan to have `limit` significantly larger than 1, we encourage relying on a limited number of workers per queue instead. For example:
 
 ```ruby
 class ThrottledJob < ApplicationJob

Rakefile

@@ -5,7 +5,9 @@ require "bundler/setup"
 APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
 load "rails/tasks/engine.rake"
 
-load "rails/tasks/statistics.rake"
+if Rails::VERSION::MAJOR < 8
+  load "rails/tasks/statistics.rake"
+end
 
 require "bundler/gem_tasks"
 require "rake/tasklib"

app/models/solid_queue/claimed_execution.rb

@@ -37,8 +37,10 @@ def release_all
     end
 
     def fail_all_with(error)
-      SolidQueue.instrument(:fail_many_claimed) do |payload|
-        includes(:job).tap do |executions|
+      includes(:job).tap do |executions|
+        return if executions.empty?
+
+        SolidQueue.instrument(:fail_many_claimed) do |payload|
           executions.each do |execution|
             execution.failed_with(error)
             execution.unblock_next_job

app/models/solid_queue/failed_execution.rb

@@ -58,8 +58,11 @@ def exception_backtrace
       end
 
       def determine_backtrace_size_limit
-        column = self.class.connection.schema_cache.columns_hash(self.class.table_name)["error"]
-        if column.limit.present?
+        column = self.class.connection_pool.with_connection do |connection|
+          connection.schema_cache.columns_hash(self.class.table_name)["error"]
+        end
+
+        if column && column.limit.present?
           column.limit - exception_class_name.bytesize - exception_message.bytesize - JSON_OVERHEAD
         end
       end

app/models/solid_queue/job.rb

@@ -10,6 +10,7 @@ class EnqueueError < StandardError; end
 
     class << self
       def enqueue_all(active_jobs)
+        active_jobs.each { |job| job.scheduled_at ||= Time.current }
         active_jobs_by_job_id = active_jobs.index_by(&:job_id)
 
         transaction do

app/models/solid_queue/record.rb

@@ -6,11 +6,19 @@ class Record < ActiveRecord::Base
 
     connects_to(**SolidQueue.connects_to) if SolidQueue.connects_to
 
-    def self.non_blocking_lock
-      if SolidQueue.use_skip_locked
-        lock(Arel.sql("FOR UPDATE SKIP LOCKED"))
-      else
-        lock
+    class << self
+      def non_blocking_lock
+        if SolidQueue.use_skip_locked
+          lock(Arel.sql("FOR UPDATE SKIP LOCKED"))
+        else
+          lock
+        end
+      end
+
+      def supports_insert_conflict_target?
+        connection_pool.with_connection do |connection|
+          connection.supports_insert_conflict_target?
+        end
       end
     end
   end

app/models/solid_queue/recurring_execution.rb

@@ -8,7 +8,7 @@ class AlreadyRecorded < StandardError; end
 
     class << self
       def create_or_insert!(**attributes)
-        if connection.supports_insert_conflict_target?
+        if supports_insert_conflict_target?
           # PostgreSQL fails and aborts the current transaction when it hits a duplicate key conflict
           # during two concurrent INSERTs for the same value of an unique index. We need to explicitly
           # indicate unique_by to ignore duplicate rows by this value when inserting