landing_job: do not pickup handover jobs in all workers (bug 2019420)

src/lando/api/legacy/workers/base.py

@@ -188,6 +188,10 @@ def _start(self, max_loops: int | None = None, *args, **kwargs):
 
         logger.info(f"{self} exited after {loops} loops.")
 
+    def get_next_job(self) -> BaseJob:
+        """Return the next job in the queue from active repositories."""
+        return self.job_type.next_job(repositories=self.active_repos).first()
+
     def loop(self):
         """Fetch jobs and processes them.
 
@@ -211,9 +215,9 @@ def loop(self):
             self.refresh_active_repos()
 
         with transaction.atomic():
-            job = self.job_type.next_job(repositories=self.active_repos).first()
+            job = self.get_next_job()
 
-        if job is None:
+        if job is None or job.status is JobStatus.DEFERRED:
             self.throttle(self.worker_instance.sleep_seconds)
             return
 

src/lando/api/legacy/workers/landing_worker.py

@@ -82,6 +82,12 @@ def notify_user_of_bug_update_failure(job: LandingJob, exception: Exception):
             job.id,
         )
 
+    def get_next_job(self) -> LandingJob:
+        """Return the next job in the queue from all enabled repositories."""
+        # NOTE: This could include DEFERRED jobs that were deferred due to a
+        # closed tree.
+        return self.job_type.next_job(repositories=self.enabled_repos).first()
+
     @override
     def run_job(self, job: LandingJob) -> bool:
         """Run a given LandingJob and return appropriate boolean state.

src/lando/api/tests/test_landings.py

@@ -1427,35 +1427,39 @@ def test_handover_landing_job(
     assert job.skip_treestatus_check
     assert job.target_repo == git_repo
 
-    next_job = git_worker.job_type.next_job(
-        repositories=git_worker.active_repos
-    ).first()
+    assert LandingJob.objects.all().count() == 1
+
+    next_git_job = git_worker.get_next_job()
+
+    next_hg_job = hg_worker.get_next_job()
+
     assert not job.is_handed_over
-    assert next_job == job
+    assert next_git_job == job
+    assert next_hg_job is None
     git_worker.start(max_loops=1)
     job.refresh_from_db()
     assert job.status == JobStatus.DEFERRED, job.error
     assert job.error == "Job deferred to try repo."
     assert job.is_handed_over
     assert job.target_repo == try_repo
     assert job.attempts == 1
-
-    next_job = hg_worker.job_type.next_job(repositories=hg_worker.active_repos).first()
     assert not job.skip_treestatus_check
 
+    next_git_job = git_worker.get_next_job()
+    next_hg_job = hg_worker.get_next_job()
+
+    assert next_git_job is None
+
     patch = job.revisions[0].patch
     expected = TRY_TASK_CONFIG_DIFF_SNIPPET.format(git_repo.normalized_url)
     assert patch.endswith(expected)
 
+    assert next_hg_job == job
+    hg_worker.start(max_loops=1)
+    job.refresh_from_db()
+    assert job.attempts == 2
+
     if "try" in closed_repos:
-        assert next_job is None
-        hg_worker.start(max_loops=1)
-        job.refresh_from_db()
         assert job.status == JobStatus.DEFERRED, job.error
-        assert job.attempts == 1
     else:
-        assert next_job == job
-        hg_worker.start(max_loops=1)
-        job.refresh_from_db()
-        assert job.attempts == 2
         assert job.status == JobStatus.LANDED, job.error

src/lando/main/models/landing_job.py

@@ -6,7 +6,7 @@
 
 from django.conf import settings
 from django.db import models
-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 from mots.config import FileConfig
 from mots.directory import Directory
 
@@ -216,12 +216,8 @@ def job_queue_query(
             grace_seconds (int): Ignore landing jobs that were submitted after this
                 many seconds ago.
         """
-        q = super().job_queue_query()
+        q = super().job_queue_query(repositories)
 
-        if repositories:
-            q = q.filter(
-                Q(target_repo__in=repositories) | Q(handover_repo__in=repositories)
-            )
         if grace_seconds:
             now = datetime.datetime.now(datetime.timezone.utc)
             grace_cutoff = now - datetime.timedelta(seconds=grace_seconds)