Force delete very old delayed jobs

We found some old jobs in the delayed_jobs table, which are beyond the cutoff_age_in_days interval.
They had either no failed_at set or were still locked. With this change the FailedJobsCleanup job will force delete all jobs,
which are older than 2 times the cutoff_age_in_days.

Author: svkrieger

app/jobs/runtime/failed_jobs_cleanup.rb

@@ -20,6 +20,18 @@ def perform
 
           old_delayed_jobs.delete
 
+          # There were some very old jobs in the table which did not get cleaned up
+          # This deletes those orphaned jobs, which were scheduled to run before double the cutoff age
+          double_cutoff_days = cutoff_age_in_days.to_i * 2
+          orphaned_delayed_jobs = Delayed::Job.
+                                  where(Sequel.lit("run_at < CURRENT_TIMESTAMP - INTERVAL '?' DAY", double_cutoff_days))
+
+          unless orphaned_delayed_jobs.count.zero?
+            logger.info("Cleaning up #{orphaned_delayed_jobs.count} Delayed Jobs older than #{double_cutoff_days} days")
+
+            orphaned_delayed_jobs.delete
+          end
+
           return if max_number_of_failed_delayed_jobs.nil?
 
           ids_exceeding_limit = Delayed::Job.

spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb

@@ -73,6 +73,41 @@ def max_attempts
                 Delayed::Job.find(id: @delayed_job.id)
               }.from(@delayed_job).to(nil)
             end
+
+            context 'when job is orphaned' do
+              it 'does not remove the job if it is not older than 2 times the cut-off' do
+                # Simulate a job that is orphaned but not older than 2 times the cut-off
+                Sequel::Model.db[:delayed_jobs].where(id: @delayed_job.id).update(failed_at: nil, locked_by: nil)
+                puts @delayed_job.inspect
+                expect do
+                  cleanup_job.perform
+                end.not_to(change { Delayed::Job.find(id: @delayed_job.id) })
+              end
+            end
+          end
+
+          context 'when a job is orphaned and older than 2 times the cut-off' do
+            let(:run_at) { Time.now.utc - 5.days }
+
+            it 'removes the job even if it is not failed and regardless of locked_by' do
+              # Simulate a job that is not failed but still locked
+              Sequel::Model.db[:delayed_jobs].where(id: @delayed_job.id).update(failed_at: nil, locked_by: 'some-worker', locked_at: Time.now.utc - 4.days)
+              expect do
+                cleanup_job.perform
+              end.to change {
+                Delayed::Job.find(id: @delayed_job.id)
+              }.from(@delayed_job).to(nil)
+            end
+
+            it 'removes the job even if it is not failed and locked_by is nil' do
+              # Simulate a job that is not failed and not locked
+              Sequel::Model.db[:delayed_jobs].where(id: @delayed_job.id).update(failed_at: nil, locked_by: nil)
+              expect do
+                cleanup_job.perform
+              end.to change {
+                Delayed::Job.find(id: @delayed_job.id)
+              }.from(@delayed_job).to(nil)
+            end
           end
 
           context 'when the number of delayed jobs exceeds max_number_of_failed_delayed_jobs' do