wb | add automatic check for node.ssd_directory cleanup during cloud deployments

Author: fmaste

nix/workbench/backend/nomad/cloud.sh

@@ -707,10 +707,10 @@ allocate-run-nomadcloud() {
          && jqtest '.node.utxo_lmdb' "${dir}"/profile.json                     \
          && jqtest '(.cluster.nomad.host_volumes.producer | length) > 0' "${dir}"/profile.json
       then
-        # Index on the "host_volumes" array.
+        # Iterate over the profile's Nomad "host_volumes" array by key/index.
         for host_volume_key in $(jq_tolist '.cluster.nomad.host_volumes.producer | keys' "${dir}"/profile.json)
         do
-          # Compare the SSD directory the node will use and volume destination.
+          # Compare defined node's SSD directory with this volume's destination.
           local ssd_directory host_volume_destination
           ssd_directory="$(jq -r '.node.ssd_directory' "${dir}"/profile.json)"
           host_volume_destination="$(jq -r ".cluster.nomad.host_volumes.producer[${host_volume_key}].destination" "${dir}"/profile.json)"
@@ -721,10 +721,16 @@ allocate-run-nomadcloud() {
             host_volume_source="$(jq -r ".cluster.nomad.host_volumes.producer[${host_volume_key}].source" "${dir}"/profile.json)"
             host_volume_path="$(wb nomad clients ssh "${client_name}" cat /etc/nomad.json | jq -r ".client.host_volume[\"${host_volume_source}\"].path")"
             local rm_command
+            # Won't remove hidden files like ".mounted".
             rm_command="rm -rf \"${host_volume_path}\"/*"
-            msg "$(yellow "About to: \"${rm_command}\"")"
-            read -p "Hit enter to continue ..."
+            msg "$(yellow "Cleaning up SSD host volume. Executing: \"${rm_command}\" on Nomad client \"${client_name}\" (${client_id}) ...")"
             wb nomad clients ssh "${client_name}" "${rm_command}"
+            # Check that the directory is empty (hidden files ignored).
+            if wb nomad clients ssh "${client_name}" "ls \"${host_volume_path}\"/" | grep --invert-match --quiet -E "^0$"
+            then
+              msg "$(yellow "WARNING: Nomad client \"${client_name}\" (${client_id}) still has files in \"${host_volume_path}\"")"
+              read -p "Hit enter to continue ..."
+            fi
           fi
         done
       fi