Merge pull request #482 from frappe/mergify/bp/master/pr-479

feat(bench): Add job to remove zombie benches (backport #479)

Author: Aradhya-Tripathi

agent/server.py

@@ -140,7 +140,7 @@ def container_exists(self, name: str, max_retries: int = 3):
         """
         for attempt in range(max_retries):
             try:
-                self.execute(f"""docker ps --all --filter "name=^{name}$" | grep {name}""")
+                self.execute(f"docker inspect {name}")
             except AgentException:
                 break  # container does not exist
             else:
@@ -201,6 +201,21 @@ def get_reclaimable_size(self) -> dict[str, dict[str, float] | float]:
             "total": round((unused_images_size + total_archived_folder_size) / 1024**3, 2),
         }
 
+    @job("Force Remove Zombie Benches")
+    def force_remove_zombie_benches(self, bench_names: list[str]):
+        return self._force_remove_zombie_benches(bench_names)
+
+    @step("Force Remove Zombie Benches")
+    def _force_remove_zombie_benches(self, bench_names: list[str]):
+        for bench_name in bench_names:
+            try:
+                self.execute(f"""docker ps --all --filter "name=^{bench_name}$" | grep {bench_name}""")
+            except AgentException:
+                continue  # Bench is gone
+
+            self.disable_production_on_bench(bench_name)
+            self._move_bench_to_archived_directory(bench_name)
+
     @job("Push Images to Registry")
     def push_images_to_registry(self, images: list[str], registry_settings: dict[str, str]) -> None:
         return self._push_images_to_registry(images, registry_settings)
@@ -456,8 +471,7 @@ def remove_unused_docker_artefacts(self):
             "after": after,
         }
 
-    @step("Move Bench to Archived Directory")
-    def move_bench_to_archived_directory(self, bench_name):
+    def _move_bench_to_archived_directory(self, bench_name):
         if not os.path.exists(self.archived_directory):
             os.mkdir(self.archived_directory)
         target = os.path.join(self.archived_directory, bench_name)
@@ -472,6 +486,10 @@ def move_bench_to_archived_directory(self, bench_name):
 
         self.execute(f"mv {bench_directory} {self.archived_directory}")
 
+    @step("Move Bench to Archived Directory")
+    def move_bench_to_archived_directory(self, bench_name):
+        self._move_bench_to_archived_directory(bench_name)
+
     @job("Update Site Pull", priority="low")
     def update_site_pull_job(self, name, source, target, activate):
         source = Bench(source, self)
@@ -803,6 +821,10 @@ def get_bench(self, bench):
         except KeyError as exc:
             raise BenchNotExistsException(bench) from exc
 
+    def get_running_bench_containers(self) -> list[str]:
+        """Get the actual containers skipping the `Server.benches` property"""
+        return self.execute("docker ps --format '{{.Names}}'")["output"].split("\n")
+
     @property
     def job_record(self):
         if self.job is None:

agent/web.py

@@ -246,6 +246,11 @@ def start_bench_workers():
     return {"job": job}
 
 
+@application.route("/server/running-benches", methods=["GET"])
+def get_running_benches():
+    return {"benches": Server().get_running_bench_containers()}
+
+
 @application.route("/server/force-remove-all-benches", methods=["POST"])
 def force_remove_all_benches():
     job = Server().force_remove_all_benches()
@@ -483,6 +488,14 @@ def archive_bench(bench):
     return {"job": job}
 
 
+@application.route("/benches/force-remove", methods=["POST"])
+def force_remove_zombie_benches():
+    data = request.json
+    benches = data.get("benches", [])
+    job = Server().force_remove_zombie_benches(benches)
+    return {"job": job}
+
+
 @application.route("/benches/<string:bench>/restart", methods=["POST"])
 @validate_bench
 def restart_bench(bench):