workers: add landing worker warning logs

landoapi/workers/base.py

@@ -152,7 +152,6 @@ def refresh_enabled_repos(self):
             for r in self.applicable_repos
             if treestatus_subsystem.client.is_open(repo_clone_subsystem.repos[r].tree)
         ]
-        logger.info(f"{len(self.enabled_repos)} enabled repos: {self.enabled_repos}")
 
     def start(self, max_loops: int | None = None):
         """Run setup sequence and start the event loop."""

landoapi/workers/landing_worker.py

@@ -86,11 +86,37 @@ def __init__(self, *args, **kwargs):
         self.last_job_finished = None
         self.refresh_enabled_repos()
 
-    def loop(self):
-        logger.debug(
-            f"{len(self.applicable_repos)} applicable repos: {self.applicable_repos}"
+    def check_landing_worker_warnings(self):
+        """Log messages that show various important statistics about the landing worker."""
+        TOO_MANY_ATTEMPTS_THRESHOLD = 10
+        # TODO: should this threshold be different for try landing worker?
+        QUEUE_SIZE_THRESHOLD = 20
+        ACTIVE_STATUSES = (
+            LandingJobStatus.SUBMITTED,
+            LandingJobStatus.DEFERRED,
+            LandingJobStatus.IN_PROGRESS,
+        )
+
+        queue_size = LandingJob.query.filter(
+            LandingJob.status.in_(ACTIVE_STATUSES)
+        ).count()
+        if queue_size >= QUEUE_SIZE_THRESHOLD:
+            logger.warning(f"The landing queue size of {queue_size} exceeds threshold")
+
+        runaway_jobs = LandingJob.query.filter(
+            LandingJob.status.in_(ACTIVE_STATUSES),
+            LandingJob.attempts >= TOO_MANY_ATTEMPTS_THRESHOLD,
         )
 
+        if runaway_jobs.count() > 0:
+            job = runaway_jobs.all()[0]
+            logger.warning(
+                f"Active landing job ({job}) has too many attempts ({job.attempts})"
+            )
+
+    def loop(self):
+        self.check_landing_worker_warnings()
+
         # Check if any closed trees reopened since the beginning of this iteration
         if len(self.enabled_repos) != len(self.applicable_repos):
             self.refresh_enabled_repos()