try to improve robustness of reset

Signed-off-by: Justin Alvarez <[email protected]>

Author: pendo324

tools/thinpool.sh

@@ -93,15 +93,79 @@ else
     }
 
     pool_remove() {
-        sudo dmsetup remove "${dm_device}-snap-"* || true
-        sudo dmsetup remove \
-         "${dm_device}" \
-         "${dm_device}_tdata" "${dm_device}_tmeta" || true
-        sudo lvremove -f "$dm_device"
+        # Find and remove individual snapshots with retry logic
+        for snapshot in $(sudo dmsetup ls | grep "^$(basename ${dm_device})-snap-" | awk '{print $1}' | sort -r); do
+            echo "Attempting to remove snapshot: $snapshot"
+            local retries=5
+            while [ $retries -gt 0 ]; do
+                if sudo dmsetup remove "$snapshot" 2>/dev/null; then
+                    echo "Successfully removed snapshot: $snapshot"
+                    break
+                else
+                    echo "Snapshot $snapshot busy, waiting... (retries left: $retries)"
+                    sleep 2
+                    retries=$((retries - 1))
+                fi
+            done
+            
+            if [ $retries -eq 0 ]; then
+                echo "Warning: Failed to remove snapshot $snapshot after retries"
+                # Force remove by checking what's using it
+                sudo dmsetup info "$snapshot" 2>/dev/null || true
+                sudo lsof "$snapshot" 2>/dev/null || true
+                sudo dmsetup remove --force "$snapshot" || true
+            fi
+        done
+
+        # Remove the thin pool with retries
+        local pool_retries=5
+        while [ $pool_retries -gt 0 ]; do
+            if sudo dmsetup remove "${dm_device}" 2>/dev/null; then
+                echo "Successfully removed thin pool: ${dm_device}"
+                break
+            else
+                echo "Thin pool ${dm_device} busy, waiting... (retries left: $pool_retries)"
+                sleep 2
+                pool_retries=$((pool_retries - 1))
+            fi
+        done
+        
+        if [ $pool_retries -eq 0 ]; then
+            echo "Warning: Failed to remove thin pool after retries"
+            sudo dmsetup info "${dm_device}" 2>/dev/null || true
+            sudo lsof "${dm_device}" 2>/dev/null || true
+        fi
+
+        # Clean up metadata and data devices
+        sudo dmsetup remove "${dm_device}_tdata" 2>/dev/null || true
+        sudo dmsetup remove "${dm_device}_tmeta" 2>/dev/null || true
+        
+        # Finally remove the logical volume
+        sudo lvremove -f "$dm_device" 2>/dev/null || true
     }
 
     pool_reset() {
         if [ -e "${dm_device}" ]; then
+            # Wait for containerd to finish any pending cleanup operations
+            echo "Waiting for containerd to finish cleanup operations..."
+            
+            # Use containerd client to clean up any remaining snapshots
+            if command -v ctr >/dev/null 2>&1; then
+                echo "Cleaning up containerd snapshots..."
+                # List and remove any active snapshots using containerd API
+                for snap in $(ctr --address /run/firecracker-containerd/containerd.sock snapshots list -q 2>/dev/null || true); do
+                    echo "Removing containerd snapshot: $snap"
+                    ctr --address /run/firecracker-containerd/containerd.sock snapshots remove "$snap" 2>/dev/null || true
+                done
+                
+                # Give containerd time to release device references
+                sleep 2
+            fi
+            
+            # Sync filesystem and drop caches to help release any remaining references
+            sync
+            echo 1 > /proc/sys/vm/drop_caches 2>/dev/null || true
+            
             pool_remove
         fi
         pool_create