diff --git a/scancodeio/worker.py b/scancodeio/worker.py index 2166965556..799db2fad7 100644 --- a/scancodeio/worker.py +++ b/scancodeio/worker.py @@ -23,7 +23,7 @@ from django.apps import apps from rq.queue import Queue -from rq.worker import Worker +from rq.worker import Worker, SimpleWorker scanpipe_app = apps.get_app_config("scanpipe") @@ -54,6 +54,36 @@ def run_maintenance_tasks(self): scanpipe_app.sync_runs_and_jobs() +class ScanCodeIOSimpleWorker(SimpleWorker): + """ + Modified version of RQ SimpleWorker including ScanCode.io customizations. + This class can be used when encountering TLS connection issues to the database, + as shown in https://github.com/aboutcode-org/scancode.io/issues/1523 + """ + + def run_maintenance_tasks(self): + """ + Add Runs and Jobs synchronization to the periodic maintenance tasks. + Maintenance tasks should run on first worker startup or every 10 minutes. + + During the maintenance, one of the worker will acquire a "cleaning lock" and + will run the registries cleanup. + During that cleanup, started Jobs that haven't sent a heartbeat in the past 90 + seconds (job_monitoring_interval + 60) will be considered failed and will be + moved to the FailedJobRegistry. + This happens when the Job process is killed (voluntary or not) and the heartbeat + is the RQ approach to determine if the job is stills active. + The `sync_runs_and_jobs` will see this Job as failed and will update its related + Run accordingly. + """ + super().run_maintenance_tasks() + + # The Runs and Jobs synchronization needs to be executed after the + # `self.clean_registries()` that takes place in the parent + # `super().run_maintenance_tasks()`. + scanpipe_app.sync_runs_and_jobs() + + class ScanCodeIOQueue(Queue): """Modified version of RQ Queue including ScanCode.io customizations."""