Force delete very old delayed jobs (#4395)

We found some old jobs in the delayed_jobs table, which are beyond the cutoff_age_in_days interval.
They had either no failed_at set or were still locked. With this change the FailedJobsCleanup job will force delete all jobs,
which are older than the cutoff_age_in_days + 1 day.

Author: svkrieger

app/jobs/runtime/failed_jobs_cleanup.rb

@@ -20,6 +20,18 @@ def perform
 
           old_delayed_jobs.delete
 
+          # There were some very old jobs in the table which did not get cleaned up
+          # This deletes those orphaned jobs, which were scheduled to run before the cutoff age + 1 day
+          force_delete_after = cutoff_age_in_days.to_i + 1
+          orphaned_delayed_jobs = Delayed::Job.
+                                  where(Sequel.lit("run_at < CURRENT_TIMESTAMP - INTERVAL '?' DAY", force_delete_after))
+
+          unless orphaned_delayed_jobs.count.zero?
+            logger.info("Deleting #{orphaned_delayed_jobs.count} orphaned Delayed Jobs older than #{force_delete_after} days")
+
+            orphaned_delayed_jobs.delete
+          end
+
           return if max_number_of_failed_delayed_jobs.nil?
 
           ids_exceeding_limit = Delayed::Job.

spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb

@@ -64,7 +64,7 @@ def max_attempts
           end
 
           context 'when older than specified cut-off' do
-            let(:run_at) { Time.now.utc - 3.days }
+            let(:run_at) { Time.now.utc - 50.hours }
 
             it 'removes the job' do
               expect do
@@ -73,6 +73,37 @@ def max_attempts
                 Delayed::Job.find(id: @delayed_job.id)
               }.from(@delayed_job).to(nil)
             end
+
+            context 'when job is orphaned' do
+              it 'does not remove the job if it is not older than cut-off + 1 day' do
+                Sequel::Model.db[:delayed_jobs].where(id: @delayed_job.id).update(failed_at: nil, locked_by: nil)
+                expect do
+                  cleanup_job.perform
+                end.not_to(change { Delayed::Job.find(id: @delayed_job.id) })
+              end
+            end
+          end
+
+          context 'when a job is orphaned and older than the cut-off + 1 day' do
+            let(:run_at) { Time.now.utc - 73.hours }
+
+            it 'removes the job even if it is not failed and regardless of locked_by' do
+              Sequel::Model.db[:delayed_jobs].where(id: @delayed_job.id).update(failed_at: nil, locked_by: 'some-worker', locked_at: Time.now.utc - 2.days)
+              expect do
+                cleanup_job.perform
+              end.to change {
+                Delayed::Job.find(id: @delayed_job.id)
+              }.from(@delayed_job).to(nil)
+            end
+
+            it 'removes the job even if it is not failed and locked_by is nil' do
+              Sequel::Model.db[:delayed_jobs].where(id: @delayed_job.id).update(failed_at: nil, locked_by: nil)
+              expect do
+                cleanup_job.perform
+              end.to change {
+                Delayed::Job.find(id: @delayed_job.id)
+              }.from(@delayed_job).to(nil)
+            end
           end
 
           context 'when the number of delayed jobs exceeds max_number_of_failed_delayed_jobs' do