Cgroup2Manager: Fix cgroup deletions (#439)

If there's any nested cgroups in the one we made for the container
(commonly seen for systemd images) removeItem didn't seem to be having a
grand time, even though it states it should do recursive removals. Lets
roll our own, and have a small EBUSY/EAGAIN retry loop as well. This
fixes LinuxContainer.stop() for any containers with nested cgs.

Context: apple/container#928

Author: dcantah

vminitd/Sources/Cgroup/Cgroup2Manager.swift

@@ -285,7 +285,44 @@ package struct Cgroup2Manager: Sendable {
         if force {
             try self.kill()
         }
-        try FileManager.default.removeItem(at: self.path)
+
+        // Recursively remove child cgroups first
+        try removeChildCgroups(at: self.path, force: force)
+
+        let result = rmdir(self.path.path)
+        if result != 0 {
+            throw Error.errno(errno: errno, message: "failed to remove cgroup directory \(self.path.path)")
+        }
+    }
+
+    private func removeChildCgroups(at path: URL, force: Bool) throws {
+        let fileManager = FileManager.default
+
+        guard let contents = try? fileManager.contentsOfDirectory(atPath: path.path) else {
+            return
+        }
+
+        // Remove child directories (potential nested cgroups) first
+        for item in contents {
+            let childPath = path.appending(path: item)
+            var isDirectory: ObjCBool = false
+
+            if fileManager.fileExists(atPath: childPath.path, isDirectory: &isDirectory) && isDirectory.boolValue {
+                if force {
+                    try Self.writeValue(
+                        path: childPath,
+                        value: "1",
+                        fileName: Self.killFile
+                    )
+                }
+
+                try removeChildCgroups(at: childPath, force: force)
+                let result = rmdir(childPath.path)
+                if result != 0 {
+                    throw Error.errno(errno: errno, message: "failed to remove child cgroup \(childPath.path)")
+                }
+            }
+        }
     }
 
     package func stats() throws -> Cgroup2Stats {

vminitd/Sources/vminitd/ManagedContainer.swift

@@ -110,6 +110,45 @@ actor ManagedContainer {
 }
 
 extension ManagedContainer {
+    // removeCgroupWithRetry will remove a cgroup path handling EAGAIN and EBUSY errors and
+    // retrying the remove after an exponential timeout
+    private func removeCgroupWithRetry() async throws {
+        var delay = 10  // 10ms
+        let maxRetries = 5
+
+        for i in 0..<maxRetries {
+            if i != 0 {
+                try await Task.sleep(for: .milliseconds(delay))
+                delay *= 2
+            }
+
+            do {
+                try self.cgroupManager.delete(force: true)
+                return
+            } catch let error as Cgroup2Manager.Error {
+                guard case .errno(let errnoValue, let message) = error,
+                    errnoValue == EBUSY || errnoValue == EAGAIN
+                else {
+                    throw error
+                }
+                self.log.warning(
+                    "cgroup deletion failed with EBUSY/EAGAIN, retrying",
+                    metadata: [
+                        "attempt": "\(i + 1)",
+                        "delay": "\(delay)",
+                        "errno": "\(errnoValue)",
+                        "context": "\(message)",
+                    ])
+                continue
+            }
+        }
+
+        throw ContainerizationError(
+            .internalError,
+            message: "cgroups: unable to remove cgroup after \(maxRetries) retries"
+        )
+    }
+
     private func ensureExecExists(_ id: String) throws {
         if self.execs[id] == nil {
             throw ContainerizationError(
@@ -184,7 +223,7 @@ extension ManagedContainer {
         // Delete the bundle and cgroup
         try self.bundle.delete()
         if self.needsCgroupCleanup {
-            try self.cgroupManager.delete(force: true)
+            try await self.removeCgroupWithRetry()
         }
     }