Add LUKS recovery key to vault

* Add LUKS recovery key to vault as part of mangoctl bootstrap or enroll
  for LUKS volumes in each node
* added validation steps in self_test.sh
* Added additional recovery_test.sh to run as part of integration test
  within run_tests.sh
* add machine id entity metadata for a node in the vault
* Add policy so node token can write (not update or read) recovery key
  in vault kv

Author: Harish Kumar

.github/workflows/build.yml

@@ -105,7 +105,6 @@ jobs:
       - name: Test it
         run: |
           #!/bin/bash
-          set -x
           set -e
           sudo apt-get update -y
           # mkosi doesn't pick this up from the tools dir for some reason

.github/workflows/pr.yml

@@ -71,7 +71,6 @@ jobs:
       - name: Test it
         run: |
           #!/bin/bash
-          set -x
           set -e
           sudo apt-get update -y
           # mkosi doesn't pick this up from the tools dir for some reason

mkosi.images/base/mkosi.build.chroot

@@ -1,5 +1,4 @@
 #!/bin/sh
-set -x
 
 cp -r ${SRCDIR}/resources/mangosctl ${BUILDDIR}
 cd ${BUILDDIR}/mangosctl

mkosi.images/base/mkosi.extra/usr/share/mangos/self_test.sh

@@ -3,7 +3,6 @@ BASE_URL=${BASE_URL:-http://10.0.2.2:8081}
 export BASE_URL
 
 set -e
-set -x
 
 trap 'journalctl -n 1000 --no-pager' ERR
 systemctl is-active [email protected]
@@ -26,3 +25,36 @@ do
         sleep 10
         echo "Trying again. $tries tries left"
 done
+
+echo "===> Validating Recovery Keys"
+machine_id=$(cat /etc/machine-id)
+
+# Auto-detect LUKS partitions
+luks_partitions=$(lsblk -nlo NAME,TYPE,FSTYPE | awk '$2 == "part" && $3 == "crypto_LUKS" {print $1}' | tr '\n' ' ')
+
+if [ -z "$luks_partitions" ]; then
+    echo "No LUKS partitions found, skipping recovery key validation"
+else
+    # Test 1: Verify recovery keys exist in Vault
+    for device in $luks_partitions; do
+        partition=$(lsblk -nlo PARTLABEL /dev/$device | tr -d '\n')
+        if ! mangosctl sudo -- vault kv get "secrets/mangos/recovery-keys/${machine_id}/${partition}" >/dev/null 2>&1; then
+            echo "ERROR: Recovery key not found in Vault for ${partition}"
+            exit 1
+        fi
+        echo "Recovery key for ${partition}: OK"
+    done
+
+    # Test 2: Verify LUKS has multiple keyslots (TPM + recovery)
+    for device in $luks_partitions; do
+        partition=$(lsblk -nlo PARTLABEL /dev/$device | tr -d '\n')
+        slots=$(cryptsetup luksDump /dev/$device 2>/dev/null | grep -c "^  [0-9]: luks2" || echo 0)
+        if [ "$slots" -lt 2 ]; then
+            echo "ERROR: ${partition} has only ${slots} keyslot(s), expected at least 2 (TPM + recovery)"
+            exit 1
+        fi
+        echo "LUKS keyslots for ${partition}: ${slots} OK"
+    done
+
+    echo "Recovery key validation: PASSED"
+fi

mkosi.images/terraform/share/terraform/pki.tf

@@ -264,6 +264,7 @@ resource "vault_cert_auth_backend_role" "node" {
       vault_policy.node-cert-self-renew.name,
       vault_policy.ssh-host-self-signer.name,
       vault_policy.consul-gossip.name,
+      vault_policy.node-recovery-keys.name,
     ]
 }
 
@@ -277,6 +278,22 @@ resource "vault_policy" "node-cert-self-renew" {
     EOP
 }
 
+resource "vault_policy" "node-recovery-keys" {
+    name = "node-recovery-keys"
+
+    policy = <<-EOP
+        # Allow nodes to create recovery keys for their own machine-id only (write-once, no read/update)
+        # No read allowed because node does not need to read its own recovery key,
+        #   it is only needed to be read by admins to recover the node
+        # No update allowed to avoid compromised node may update recovery key
+        #  Any update of recovery keys (even for rotating recovery keys) need admin actions
+        #  For which admin key should be used
+        path "secrets/mangos/recovery-keys/{{identity.entity.metadata.machine_id}}/*" {
+            capabilities = ["create"]
+        }
+    EOP
+}
+
 resource "vault_identity_group" "vault-servers" {
   name        = "vault-servers"
   type        = "internal"

recovery_test.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+machine_id=$(cat /etc/machine-id)
+
+# Auto-detect first LUKS partition for testing
+test_device=$(lsblk -nlo NAME,TYPE,FSTYPE | awk '$2 == "part" && $3 == "crypto_LUKS" {print $1; exit}')
+if [ -z "$test_device" ]; then
+    echo "No LUKS partitions found, skipping recovery test"
+    exit 0
+fi
+
+test_partition=$(lsblk -nlo PARTLABEL /dev/$test_device | tr -d '\n')
+mapper_name=$(lsblk -nlo NAME /dev/$test_device | awk 'NR==2')
+
+echo "Testing recovery unlock for ${test_partition} (device: ${test_device}, mapper: ${mapper_name})..."
+
+# Get recovery key from Vault
+recovery_key=$(mangosctl sudo -- vault kv get -field=key "secrets/mangos/recovery-keys/${machine_id}/${test_partition}")
+
+# Find TPM keyslot number
+tpm_slot=$(cryptsetup luksDump /dev/$test_device | \
+    awk '/Tokens:/,/Keyslots:/ {if (/systemd-tpm2/) found=1; if (found && /^  [0-9]+:/) {print $1; exit}}' | \
+    tr -d ':')
+
+echo "Removing TPM keyslot ${tpm_slot} (simulating TPM failure)..."
+PASSWORD="$recovery_key" systemd-cryptenroll --wipe-slot=tpm2 /dev/$test_device
+
+
+# Get mount point for this partition
+mount_point=$(findmnt -n -o TARGET /dev/mapper/$mapper_name)
+
+# Unmount and close
+if [ -n "$mount_point" ]; then
+    systemctl stop $(systemd-escape -p --suffix=mount "$mount_point")
+fi
+cryptsetup close $mapper_name
+
+# THE CRITICAL TEST: Unlock with recovery key
+echo "Unlocking with recovery key..."
+echo -n "$recovery_key" | systemd-cryptsetup attach $mapper_name /dev/$test_device -
+
+# Remount
+if [ -n "$mount_point" ]; then
+    mount /dev/mapper/$mapper_name "$mount_point"
+fi
+
+# Verify device is accessible
+if [ ! -b /dev/mapper/$mapper_name ]; then
+    echo "ERROR: Device not accessible after recovery"
+    exit 1
+fi
+
+echo "Data accessible after recovery: OK"
+
+# Re-enroll TPM (cleanup for future tests)
+echo "Re-enrolling TPM keyslot..."
+echo -n "$recovery_key" | systemd-cryptenroll /dev/$test_device \
+    --tpm2-device=auto \
+    --tpm2-pcrs=7 \
+    --tpm2-public-key-pcrs=11 \
+    --unlock-key-file=/dev/stdin
+
+echo "Recovery test: PASSED"

resources/mangosctl/mangosctl.sh

@@ -320,6 +320,62 @@ do_install() {
 	fi
 }
 
+# Enroll recovery keys for encrypted partitions and store them in Vault
+enroll_recovery_keys() {
+	local vault_token="$1"
+	local machine_id="$(cat /etc/machine-id)"
+	local found_any=0
+
+	# Find all LUKS-encrypted partitions
+	local devices=($(lsblk -ln -o NAME,TYPE,FSTYPE | awk '$2=="part" && $3=="crypto_LUKS" {print "/dev/"$1}'))
+
+	for device in "${devices[@]}"; do
+		local partlabel=$(lsblk -n -o PARTLABEL "$device" 2>/dev/null | tr -d ' \n\r\t')
+
+		# Skip if no valid partition label
+		if [ -z "$partlabel" ]; then
+			continue
+		fi
+
+		# Skip if recovery key already exists in Vault
+		if VAULT_TOKEN="${vault_token}" vault kv get "secrets/mangos/recovery-keys/${machine_id}/${partlabel}" >/dev/null 2>&1; then
+			continue
+		fi
+
+		found_any=1
+		step "Enrolling recovery key for ${partlabel}"
+
+		# Generate and enroll recovery key (systemd-cryptenroll generates and prints the key)
+		# Use TPM to unlock the device, then enroll a new recovery key
+		local output=$(systemd-cryptenroll "${device}" --recovery-key --unlock-tpm2-device=auto 2>&1)
+
+		# Extract recovery key - format: 6 lowercase alphanumeric groups of 8, separated by dashes
+		# Example: etklvner-lblhnbgl-kdtnujtk-ikjlgbur-lnlrjrrc-iuikkidg-feientnn-dkjeeuft
+		LUKS_RECOVERY_KEY_REGEX='[a-z0-9]{8}(-[a-z0-9]{8}){7}'
+		local recovery_key=$(echo "$output" | grep -oE "${LUKS_RECOVERY_KEY_REGEX}" | head -n 1)
+
+		if [ -n "$recovery_key" ] && [[ "$recovery_key" =~ ^${LUKS_RECOVERY_KEY_REGEX}$ ]]; then
+			VAULT_TOKEN="${vault_token}" vault kv put "secrets/mangos/recovery-keys/${machine_id}/${partlabel}" \
+				key="${recovery_key}" hostname="${HOSTNAME}" device="${device}" created="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+			if [ $? -eq 0 ]; then
+				greenln Success
+			else
+				red "Failed to store in Vault"
+				echo
+			fi
+		else
+			red "Failed to enroll or extract recovery key"
+			echo
+		fi
+	done
+
+	if [ $found_any -eq 0 ]; then
+		echo " > All recovery keys already enrolled"
+	else
+		echo " > Recovery keys enrolled and stored in Vault"
+	fi
+}
+
 do_enroll() {
 	declare -A groups
 
@@ -433,6 +489,11 @@ do_enroll() {
 	entity_name=$(vault write -field=name identity/lookup/entity alias_name=${HOSTNAME}.mangos alias_mount_accessor=${node_auth_accessor})
 	echo $entity_name
 
+	step "Setting machine-id as entity metadata"
+	machine_id=$(cat /etc/machine-id)
+	vault write identity/entity/name/${entity_name} metadata=machine_id="${machine_id}"
+	greenln Success
+
 	for group in ${!groups[@]}
 	do
 		do_step "Adding host to group '${group}'" chronic do_entity addgroup ${entity_name} ${group}
@@ -553,6 +614,8 @@ do_enroll() {
 	greenln Success
 
 	do_step "Reloading confexts" chronic systemd-confext refresh --mutable=auto
+
+	do_step "Enrolling recovery keys for encrypted partitions" enroll_recovery_keys "${NODE_VAULT_TOKEN}"
 }
 
 do_group() {
@@ -987,6 +1050,12 @@ do_bootstrap() {
 	NOMAD_TOKEN="${nomad_mgmt_token}" \
 	CONSUL_HTTP_TOKEN=${consul_mgmt_token} \
 	do_step "Final Terraform run" run_terraform_apply
+
+	echo
+	echo "Bootstrap complete! Next steps:"
+	echo "  1. Run: mangosctl sudo enroll -g vault-server -g consul-server -g nomad-server 127.0.0.1"
+	echo "  2. This will enroll the bootstrap node's identity and recovery keys"
+	echo
 }
 
 set_agent_token() {

run_tests.sh

@@ -258,16 +258,18 @@ chmod +x "${tmpdir}/is_ready.sh"
 
 # Exit status 130 means killed by signal 2 (SIGINT)
 step 'Waiting for installed OS to be ready'
-$systemd_run -u "mangos-test-${testid}-socat" -d -p SuccessExitStatus=130 -q --wait -- mkosi --debug sandbox -- socat VSOCK-LISTEN:23433,fork,socktype=5 EXEC:"${tmpdir}/is_ready.sh"
-report_outcome
+$systemd_run -u "mangos-test-${testid}-socat" -d -p SuccessExitStatus=130 -q --wait -- \
+    mkosi --debug sandbox -- socat VSOCK-LISTEN:23433,fork,socktype=5 EXEC:"${tmpdir}/is_ready.sh"
+
 
 step ssh into VM
+
 if $systemd_run -d --wait -q -p StandardOutput=journal -- ssh -i ./mkosi.key \
-    -o UserKnownHostsFile=/dev/null \
-    -o StrictHostKeyChecking=no \
-    -o LogLevel=ERROR \
-    -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p" \
-    root@mkosi 'mangosctl --base-url=http://10.0.2.2:8081 updatectl add-overrides ; /usr/share/mangos/self_test.sh'
+        -o UserKnownHostsFile=/dev/null \
+        -o StrictHostKeyChecking=no \
+        -o LogLevel=ERROR \
+        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p" \
+        root@mkosi 'mangosctl --base-url=http://10.0.2.2:8081 updatectl add-overrides ; /usr/share/mangos/self_test.sh'
 then
     success
     $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} succeeded"
@@ -276,3 +278,19 @@ else
     $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} failed"
     exit 1
 fi
+
+step 'Testing LUKS recovery functionality'
+if $systemd_run -d --wait -q -p StandardOutput=journal -- ssh -i ./mkosi.key \
+        -o UserKnownHostsFile=/dev/null \
+        -o StrictHostKeyChecking=no \
+        -o LogLevel=ERROR \
+        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p" \
+        root@mkosi bash -s < ./recovery_test.sh
+then
+    success
+    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} succeeded"
+else
+    failure
+    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Recovery test failed"
+    exit 1
+fi