fix nomad issue

Author: Harish Kumar

.github/workflows/pr.yml

@@ -75,6 +75,8 @@ jobs:
       - name: Export image version for later steps
         run: echo IMAGE_VERSION="$(./mkosi.version)" >> $GITHUB_ENV
       - name: Test it
+        env:
+          VERBOSE: "1"
         run: |
           #!/bin/bash
           set -x

hashiext-download.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+set -x
 VAULT_VERSION=${VAULT_VERSION:-latest}
 CONSUL_VERSION=${CONSUL_VERSION:-latest}
 NOMAD_VERSION=${NOMAD_VERSION:-latest}
@@ -18,21 +19,22 @@ download() {
         version=$(get_latest_version "$name")
     fi
 
-    version="${version#v}"
+    version="${version#v}"  # Remove leading 'v' if present
+    version="${version% *}" # sometimes version string gets extra characters with space
 
     origdir="$(pwd)"
     tmpdir=$(mktemp -d)
     cd "$tmpdir" || exit 1
 
     local url="https://releases.hashicorp.com/${name}/${version}/${name}_${version}_linux_amd64.zip"
     local fname="${url##*/}"
-    wget -O "${fname}" "${url}"
+    wget --no-verbose -O "${fname}" "${url}"
 
     sha256sums=https://releases.hashicorp.com/${name}/${version}/${name}_${version}_SHA256SUMS
     sha256sums_sig=https://releases.hashicorp.com/${name}/${version}/${name}_${version}_SHA256SUMS.sig
 
-    wget -O SHA256SUMS "${sha256sums}"
-    wget -O SHA256SUMS.sig "${sha256sums_sig}"
+    wget --no-verbose -O SHA256SUMS "${sha256sums}"
+    wget --no-verbose -O SHA256SUMS.sig "${sha256sums_sig}"
 
     if ! gpg --verify --no-default-keyring --keyring ${origdir}/resources/hashicorp-signing-key.72D7468F.gpg SHA256SUMS.sig SHA256SUMS
     then
@@ -68,7 +70,7 @@ cni_plugins=https://github.com/containernetworking/plugins/releases/download/v1.
 
 if ! [ -e "$(basename $cni_plugins)" ]
 then
-    wget $cni_plugins
+    wget --no-verbose $cni_plugins
 fi
 
 if ! [ -d resources/cni ]

mkosi.images/base/mkosi.extra/usr/share/mangos/self_test.sh

@@ -12,21 +12,66 @@ systemctl is-active systemd-cryptsetup@var.service
 systemctl is-active systemd-cryptsetup@var\\x2dtmp.service
 mangosctl bootstrap
 mangosctl sudo enroll -g{vault-server,{nomad,consul}-{server,client}}s 127.0.0.1
-mangosctl sudo -- nomad job run /usr/share/mangos/test.nomad
-tries=10
-while ! mangosctl sudo -- nomad alloc logs -namespace=admin -task server -job test | grep SUCCESS
+mangosctl sudo -- nomad job run -detach /usr/share/mangos/test.nomad
+
+echo "Waiting for job allocation to start..."
+echo "Current time: $(date)"
+tries=60
+success=0
+
+# Temporarily disable exit-on-error for polling loop
+set +e
+
+while [ $tries -gt 0 ]
 do
-        if [ $tries -le 0 ]
+        # Get allocation status first
+        alloc_status=$(mangosctl sudo -- nomad job allocs -namespace=admin -json test 2>/dev/null | jq -r '.[0].ClientStatus // empty')
+        
+        if [ -n "$alloc_status" ]; then
+                echo "[$(date +%H:%M:%S)] Allocation status: $alloc_status"
+        else
+                echo "[$(date +%H:%M:%S)] No allocation yet..."
+        fi
+        
+        # Check if logs are available and contain SUCCESS
+        if mangosctl sudo -- nomad alloc logs -namespace=admin -task server -job test 2>/dev/null | grep -q SUCCESS
         then
-                echo "Test job did not complete successfully"
-                exit 1
+                echo "Test job completed successfully!"
+                success=1
+                break
+        fi
+        
+        # If allocation failed, break early
+        if [ "$alloc_status" = "failed" ]; then
+                echo "Allocation failed, breaking loop"
+                break
         fi
+        
         tries=$((tries - 1))
-        echo "Sleeping 10 seconds."
+        echo "[$(date +%H:%M:%S)] Waiting... ($tries attempts remaining)"
         sleep 10
-        echo "Trying again. $tries tries left"
 done
 
+# Re-enable exit-on-error
+set -e
+
+if [ $success -eq 0 ]
+then
+        echo "Test job did not complete successfully after 10 minutes"
+        echo "=== Job Status ==="
+        mangosctl sudo -- nomad job status -namespace=admin test || true
+        echo "=== Allocation Logs ==="
+        mangosctl sudo -- nomad alloc logs -namespace=admin -task server -job test 2>&1 || true
+        echo "=== Allocation Status ==="
+        alloc_id=$(mangosctl sudo -- nomad job allocs -namespace=admin -json test 2>/dev/null | jq -r '.[0].ID // empty')
+        if [ -n "$alloc_id" ]; then
+                mangosctl sudo -- nomad alloc status -namespace=admin "$alloc_id" || true
+        else
+                echo "No allocations found for job"
+        fi
+        exit 1
+fi
+
 echo "===> Validating Recovery Keys"
 machine_id=$(cat /etc/machine-id)
 
@@ -38,7 +83,7 @@ if [ -z "$luks_partitions" ]; then
 else
     # Test 1: Verify recovery keys exist in Vault
     for device in $luks_partitions; do
-        partition=$(lsblk -nlo PARTLABEL /dev/$device)
+        partition=$(lsblk -n -o PARTLABEL "/dev/$device" 2>/dev/null | tr -d ' \n\r\t')
         if ! mangosctl sudo -- vault kv get "secrets/mangos/recovery-keys/${machine_id}/${partition}" >/dev/null 2>&1; then
             echo "ERROR: Recovery key not found in Vault for ${partition}"
             exit 1

mkosi.images/consul/mkosi.version

@@ -1 +1 @@
-1.22.1
+1.22.0

resources/mangosctl/mangosctl.sh

@@ -4,7 +4,6 @@ DEFAULT_REGION=global
 DEFAULT_DATACENTER=dc1
 
 set -e
-set -x
 
 usage() {
 	echo 'Usage: $0 [GLOBAL OPTIONS] {install|update|enroll}'
@@ -302,10 +301,10 @@ enroll_recovery_keys() {
 	local found_any=0
 
 	# Find all LUKS-encrypted partitions
-	local devices=($(lsblk -ln -o NAME,TYPE,FSTYPE | awk '$2=="part" && $3=="crypto_LUKS" {print "/dev/"$1}'))
+	local devices=($(lsblk -ln -o NAME,TYPE,FSTYPE | awk '$2=="part" && $3=="crypto_LUKS" {print $1}'))
 
 	for device in "${devices[@]}"; do
-		local partlabel=$(lsblk -n -o PARTLABEL "$device" 2>/dev/null | tr -d ' \n\r\t')
+		local partlabel=$(lsblk -n -o PARTLABEL "/dev/$device" 2>/dev/null | tr -d ' \n\r\t')
 
 		# Skip if no valid partition label
 		if [ -z "$partlabel" ]; then

run_tests.sh

@@ -37,22 +37,28 @@ report_outcome() {
     fi
 }
 
-# Run a command with journalctl streaming
+# Run a command with journalctl streaming (controlled by VERBOSE)
 run_with_logs() {
     local unit_filter="$1"
     local header="$2"
     shift 2
 
-    echo
-    echo "$(bold "$header")"
-    journalctl --user -u "$unit_filter" -f --no-pager &
-    local journal_pid=$!
+    local journal_pid=""
+    
+    if [ -n "${VERBOSE}" ]; then
+        echo
+        echo "$(bold "$header")"
+        journalctl --user -u "$unit_filter" -f --no-pager &
+        journal_pid=$!
+    fi
 
     "$@"
     local result=$?
 
-    kill $journal_pid 2>/dev/null || true
-    wait $journal_pid 2>/dev/null || true
+    if [ -n "$journal_pid" ]; then
+        kill $journal_pid 2>/dev/null || true
+        wait $journal_pid 2>/dev/null || true
+    fi
 
     return $result
 }