moving recovery_test.sh to image

Author: Harish Kumar

mkosi.images/base/mkosi.conf

@@ -37,6 +37,9 @@ Packages=
     # CA certificates
     ca-certificates
 
+    # LUKS support
+    cryptsetup
+
     # systemd's DNS resolver
     systemd-resolved
 

mkosi.images/base/mkosi.extra/usr/share/mangos/recovery_test.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -e
+set -x
+
+machine_id="$(cat /etc/machine-id)"
+
+test_recovery_key() {
+    test_device="$1"
+
+    test_partition="$(lsblk -nlo PARTLABEL ${test_device} | tr -d '\n')"
+    mapper_name="$(lsblk -nlo NAME ${test_device} | awk 'NR==2')"
+
+    echo "Testing recovery unlock for ${test_partition} (device: ${test_device}, mapper: ${mapper_name})..."
+    echo ""
+
+    # Get recovery key from Vault
+    recovery_key="$(mangosctl sudo -- vault kv get -field=key "secrets/mangos/recovery-keys/${machine_id}/${test_partition}")"
+
+    # Find TPM keyslot number
+    tpm_slot="$(cryptsetup luksDump ${test_device} | \
+        awk '/Tokens:/,/Keyslots:/ {if (/systemd-tpm2/) found=1; if (found && /^  [0-9]+:/) {print $1; exit}}' | \
+        tr -d ':')"
+
+    echo "Removing TPM keyslot ${tpm_slot} (simulating TPM failure)..."
+    # Provide the recovery key on stdin so systemd-cryptenroll does not prompt interactively.
+    # Use --unlock-key-file=/dev/stdin to read the key from stdin when wiping the TPM slot.
+    printf '%s' "$recovery_key" | systemd-cryptenroll --wipe-slot=tpm2 --unlock-key-file=/dev/stdin ${test_device}
+
+
+    # Get mount point for this partition
+    mount_point="$(findmnt -n -o TARGET /dev/mapper/${mapper_name})"
+
+    # Unmount and close
+    if [ -n "${mount_point}" ]; then
+        systemctl stop "$(systemd-escape -p --suffix=mount "${mount_point}")"
+    fi
+    cryptsetup close "${mapper_name}"
+
+    # THE CRITICAL TEST: Unlock with recovery key
+    echo "Unlocking with recovery key..."
+    echo -n "$recovery_key" | systemd-cryptsetup attach "${mapper_name}" "${test_device}" -
+    # Remount
+    if [ -n "${mount_point}" ]; then
+        mount /dev/mapper/"${mapper_name}" "${mount_point}"
+    fi
+
+    # Verify device is accessible
+    if [ ! -b /dev/mapper/"${mapper_name}" ]; then
+        echo "ERROR: Device not accessible after recovery"
+        exit 1
+    fi
+
+    echo "Data accessible after recovery: OK"
+
+    # Re-enroll TPM (cleanup for future tests)
+    echo "Re-enrolling TPM keyslot..."
+    # Re-enroll by supplying the recovery key on stdin (non-interactive)
+    printf '%s' "${recovery_key}" | systemd-cryptenroll "${test_device}" \
+        --tpm2-device=auto \
+        --tpm2-pcrs=7 \
+        --tpm2-public-key-pcrs=11 \
+        --unlock-key-file=/dev/stdin
+
+    echo "Recovery test: PASSED"
+
+}
+
+# Auto-detect first LUKS partition for testing
+devices="$(lsblk -ln -o NAME,TYPE,FSTYPE | awk '$2=="part" && $3=="crypto_LUKS" {print "/dev/"$1}' | tr '\n' ' ')"
+
+echo "> LUKS-encrypted devices found: $devices"
+
+for test_device in ${devices}; do
+    echo "> Testing device: ${test_device}"
+    test_recovery_key "${test_device}"
+done
+
+echo "> All recovery tests completed successfully."
+echo "failing to see whats happening"
+exit 1

mkosi.images/base/mkosi.extra/usr/share/mangos/self_test.sh

@@ -58,3 +58,14 @@ else
 
     echo "Recovery key validation: PASSED"
 fi
+
+echo 'Testing LUKS recovery functionality'
+
+if /usr/share/mangos/recovery_test.sh; then
+    echo "LUKS recovery test: PASSED"
+else
+    echo "LUKS recovery test: FAILED"
+    exit 1
+fi
+
+echo "All self-tests completed successfully."

recovery_test.sh

@@ -326,19 +326,28 @@ enroll_recovery_keys() {
 	local machine_id="$(cat /etc/machine-id)"
 	local found_any=0
 
+	local marker_dir="/var/lib/mangos/luks-recovery-keys-enrolled"
+	mkdir -p "${marker_dir}"
+
 	# Find all LUKS-encrypted partitions
-	local devices=($(lsblk -ln -o NAME,TYPE,FSTYPE | awk '$2=="part" && $3=="crypto_LUKS" {print "/dev/"$1}'))
+	local devices="$(lsblk -ln -o NAME,TYPE,FSTYPE | awk '$2=="part" && $3=="crypto_LUKS" {print "/dev/"$1}' | tr '\n' ' ')"
+
+	echo "> LUKS-encrypted devices found: $devices"
 
-	for device in "${devices[@]}"; do
-		local partlabel=$(lsblk -n -o PARTLABEL "$device" 2>/dev/null | tr -d ' \n\r\t')
+	for device in ${devices}; do
+		echo "> Processing device: ${device}"
+		local partlabel="$(lsblk -n -o PARTLABEL "${device}" 2>/dev/null | tr -d ' \n\r\t')"
 
 		# Skip if no valid partition label
-		if [ -z "$partlabel" ]; then
+		if [ -z "${partlabel}" ]; then
+			echo "> Device ${device} has no PARTLABEL, skipping"
 			continue
 		fi
 
-		# Skip if recovery key already exists in Vault
-		if VAULT_TOKEN="${vault_token}" vault kv get "secrets/mangos/recovery-keys/${machine_id}/${partlabel}" >/dev/null 2>&1; then
+		# Skip if already enrolled
+		local marker_file="${marker_dir}/${partlabel}"
+		if [ -f "${marker_file}" ]; then
+			echo "> Recovery key for ${partlabel} already enrolled, skipping"
 			continue
 		fi
 
@@ -352,30 +361,54 @@ enroll_recovery_keys() {
 		# Extract recovery key - format: 6 lowercase alphanumeric groups of 8, separated by dashes
 		# Example: etklvner-lblhnbgl-kdtnujtk-ikjlgbur-lnlrjrrc-iuikkidg-feientnn-dkjeeuft
 		LUKS_RECOVERY_KEY_REGEX='[a-z0-9]{8}(-[a-z0-9]{8}){7}'
-		local recovery_key=$(echo "$output" | grep -oE "${LUKS_RECOVERY_KEY_REGEX}" | head -n 1)
+		local recovery_key="$(echo "$output" | grep -oE "${LUKS_RECOVERY_KEY_REGEX}" | head -n 1)"
 
-		if [ -n "$recovery_key" ] && [[ "$recovery_key" =~ ^${LUKS_RECOVERY_KEY_REGEX}$ ]]; then
-			VAULT_TOKEN="${vault_token}" vault kv put "secrets/mangos/recovery-keys/${machine_id}/${partlabel}" \
+		if [ -n "${recovery_key}" ]; then
+			if VAULT_TOKEN="${vault_token}" vault kv put "secrets/mangos/recovery-keys/${machine_id}/${partlabel}" \
 				key="${recovery_key}" hostname="${HOSTNAME}" device="${device}" created="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-			if [ $? -eq 0 ]; then
+			then
 				greenln Success
+				touch "${marker_file}"
 			else
 				red "Failed to store in Vault"
-				echo
 			fi
 		else
-			red "Failed to enroll or extract recovery key"
-			echo
+			red "Failed to extract recovery key. cryptenroll output:"
+			echo "${output}"
 		fi
 	done
 
-	if [ $found_any -eq 0 ]; then
-		echo " > All recovery keys already enrolled"
+	if [ ${found_any} -eq 0 ]; then
+		echo "> All recovery keys already enrolled"
 	else
-		echo " > Recovery keys enrolled and stored in Vault"
+		echo "> Recovery keys enrolled and stored in Vault"
 	fi
 }
 
+write_machine_id_metadata() {
+	step "Getting mount accessor for node-cert"
+	node_auth_accessor="$(vault read -field=accessor sys/auth/node-cert)"
+
+	step "Looking up entity name for this node"
+	entity_name="$(vault write -field=name identity/lookup/entity alias_name=${HOSTNAME}.mangos alias_mount_accessor=${node_auth_accessor})"
+
+	step "Setting machine-id as entity metadata"
+	machine_id="$(cat /etc/machine-id)"
+
+	# Read current metadata, merge with new machine_id, and write back
+	current_metadata="$(vault read -format=json identity/entity/name/${entity_name} | jq -r '.data.metadata // {}')"
+	new_metadata="$(echo "${current_metadata}" | jq --arg mid "${machine_id}" '. + {machine_id: $mid}')"
+
+	# Convert JSON to key=value arguments for Vault CLI
+	metadata_args=()
+	while IFS='=' read -r k v; do
+		metadata_args+=("metadata=${k}=${v}")
+	done < <(echo "${new_metadata}" | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]')
+
+	vault write identity/entity/name/"${entity_name}" "${metadata_args[@]}"
+	greenln Success
+}
+
 do_enroll() {
 	declare -A groups
 
@@ -481,18 +514,7 @@ do_enroll() {
 	NODE_VAULT_TOKEN=$(vault login -method=cert -path=node-cert -client-cert=/var/lib/mangos/mangos.crt -client-key=<(systemd-creds decrypt ${confext_dir}/etc/credstore.encrypted/mangos.key) -token-only)
 	greenln Success
 
-	step "Getting mount accessor for node-cert"
-	node_auth_accessor=$(vault read -field=accessor sys/auth/node-cert)
-	echo $node_auth_accessor
-
-	step "Looking up entity name for this node"
-	entity_name=$(vault write -field=name identity/lookup/entity alias_name=${HOSTNAME}.mangos alias_mount_accessor=${node_auth_accessor})
-	echo $entity_name
-
-	step "Setting machine-id as entity metadata"
-	machine_id=$(cat /etc/machine-id)
-	vault write identity/entity/name/${entity_name} metadata=machine_id="${machine_id}"
-	greenln Success
+	do_step "Writing machine ID metadata to Vault" write_machine_id_metadata
 
 	for group in ${!groups[@]}
 	do

resources/mangosctl/mangosctl.sh

@@ -264,33 +264,59 @@ $systemd_run -u "mangos-test-${testid}-socat" -d -p SuccessExitStatus=130 -q --w
 
 step ssh into VM
 
-if $systemd_run -d --wait -q -p StandardOutput=journal -- ssh -i ./mkosi.key \
-        -o UserKnownHostsFile=/dev/null \
-        -o StrictHostKeyChecking=no \
-        -o LogLevel=ERROR \
-        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p" \
-        root@mkosi 'mangosctl --base-url=http://10.0.2.2:8081 updatectl add-overrides ; /usr/share/mangos/self_test.sh'
-then
-    success
-    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} succeeded"
-else
-    failure
-    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} failed"
-    exit 1
-fi
+# Stream the remote self-test live to the workflow console and also save to a logfile
+diag_ssh_out="${tmpdir}/self_test_ssh.out"
+echo "Streaming remote self-test output to ${diag_ssh_out}"
+
+# Use direct ssh (with forced tty) so output is streamed live. Save output with tee.
+# Run ssh+tee in background and tail the logfile in foreground so CI logs show live output
+ssh_cmd=(ssh -tt -i ./mkosi.key
+        -o UserKnownHostsFile=/dev/null
+        -o StrictHostKeyChecking=no
+        -o LogLevel=ERROR
+        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p"
+        root@mkosi "bash -lc 'mangosctl --base-url=http://10.0.2.2:8081 updatectl add-overrides ; /usr/share/mangos/self_test.sh'")
+
+# Ensure diag file exists
+touch "${diag_ssh_out}"
+
+# Trap to clean child processes on exit
+cleanup_ssh_tail() {
+    if [ -n "${ssh_pid:-}" ]; then
+        kill "${ssh_pid}" 2>/dev/null || true
+    fi
+    if [ -n "${tail_pid:-}" ]; then
+        kill "${tail_pid}" 2>/dev/null || true
+    fi
+}
+trap cleanup_ssh_tail EXIT
 
-step 'Testing LUKS recovery functionality'
-if $systemd_run -d --wait -q -p StandardOutput=journal -- ssh -i ./mkosi.key \
-        -o UserKnownHostsFile=/dev/null \
-        -o StrictHostKeyChecking=no \
-        -o LogLevel=ERROR \
-        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p" \
-        root@mkosi bash -s < ./recovery_test.sh
-then
+# Start ssh pipeline in background, using stdbuf to avoid buffering
+stdbuf -oL "${ssh_cmd[@]}" 2>&1 | stdbuf -oL tee "${diag_ssh_out}" &
+ssh_pid=$!
+
+# Give ssh/tee a moment to start writing, then tail the logfile to stream live output
+sleep 1
+tail -n +1 -f "${diag_ssh_out}" &
+tail_pid=$!
+
+# Wait for ssh to finish
+wait ${ssh_pid}
+ssh_rc=$?
+
+# Stop tailing
+kill ${tail_pid} 2>/dev/null || true
+wait ${tail_pid} 2>/dev/null || true
+
+trap - EXIT
+
+if [ ${ssh_rc} -eq 0 ]; then
     success
-    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} succeeded"
+    echo "Mangos test ${testid} succeeded" | $systemd_run -q -u "mangos-test-${testid}-result" -- cat
 else
     failure
-    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Recovery test failed"
-    exit 1
+    echo "Mangos test ${testid} failed" | $systemd_run -q -u "mangos-test-${testid}-result" -- cat
+    echo "--- Tail of remote self-test output (last 200 lines) ---"
+    tail -n 200 "${diag_ssh_out}" || true
+    exit ${ssh_rc}
 fi