stream output as the script run

Author: Harish Kumar

mkosi.images/base/mkosi.extra/usr/share/mangos/recovery_test.sh

@@ -25,7 +25,9 @@ tpm_slot=$(cryptsetup luksDump /dev/$test_device | \
     tr -d ':')
 
 echo "Removing TPM keyslot ${tpm_slot} (simulating TPM failure)..."
-PASSWORD="$recovery_key" systemd-cryptenroll --wipe-slot=tpm2 /dev/$test_device
+# Provide the recovery key on stdin so systemd-cryptenroll does not prompt interactively.
+# Use --unlock-key-file=/dev/stdin to read the key from stdin when wiping the TPM slot.
+printf '%s' "$recovery_key" | systemd-cryptenroll --wipe-slot=tpm2 --unlock-key-file=/dev/stdin /dev/$test_device
 
 
 # Get mount point for this partition
@@ -56,7 +58,8 @@ echo "Data accessible after recovery: OK"
 
 # Re-enroll TPM (cleanup for future tests)
 echo "Re-enrolling TPM keyslot..."
-echo -n "$recovery_key" | systemd-cryptenroll /dev/$test_device \
+# Re-enroll by supplying the recovery key on stdin (non-interactive)
+printf '%s' "$recovery_key" | systemd-cryptenroll /dev/$test_device \
     --tpm2-device=auto \
     --tpm2-pcrs=7 \
     --tpm2-public-key-pcrs=11 \

mkosi.images/base/mkosi.extra/usr/share/mangos/self_test.sh

@@ -61,7 +61,30 @@ fi
 
 echo 'Testing LUKS recovery functionality'
 
-timeout 120 /usr/share/mangos/recovery_test.sh
+# Run recovery_test with verbose tracing and capture output for debugging when it hangs.
+# Use a temp logfile under /var/log so it persists in the VM for inspection.
+diag_log=/var/log/recovery_test.log
+rm -f "$diag_log" || true
+echo "Starting recovery_test at $(date -u +%FT%T%z)" > "$diag_log"
+
+# Run the test with bash -x under timeout; capture both stdout and stderr.
+# We avoid `set -e` killing the script immediately so we can dump logs on failure.
+set +e
+timeout 120 bash -x /usr/share/mangos/recovery_test.sh >> "$diag_log" 2>&1
+rc=$?
+set -e
+
+echo "recovery_test exited with code: $rc" >> "$diag_log"
+echo "--- Last 200 lines of recovery_test log ---"
+tail -n 200 "$diag_log" || true
+
+echo "--- Relevant journal entries (cryptsetup, systemd-cryptenroll, verity) ---"
+journalctl -n 200 --no-pager -u systemd-cryptsetup@* -u systemd-veritysetup@* -u systemd-cryptenroll.service 2>/dev/null || true
+
+if [ $rc -ne 0 ]; then
+    echo "Recovery test failed (rc=$rc); see $diag_log inside the VM for full output"
+    exit $rc
+fi
 
 #if /usr/share/mangos/recovery_test.sh; then
     echo "LUKS recovery test: PASSED"

recovery_test.sh

@@ -24,7 +24,9 @@ tpm_slot=$(cryptsetup luksDump /dev/$test_device | \
     tr -d ':')
 
 echo "Removing TPM keyslot ${tpm_slot} (simulating TPM failure)..."
-PASSWORD="$recovery_key" systemd-cryptenroll --wipe-slot=tpm2 /dev/$test_device
+# Provide the recovery key on stdin so systemd-cryptenroll does not prompt interactively.
+# Use --unlock-key-file=/dev/stdin to read the key from stdin when wiping the TPM slot.
+printf '%s' "$recovery_key" | systemd-cryptenroll --wipe-slot=tpm2 --unlock-key-file=/dev/stdin /dev/$test_device
 
 
 # Get mount point for this partition

run_tests.sh

@@ -264,17 +264,59 @@ $systemd_run -u "mangos-test-${testid}-socat" -d -p SuccessExitStatus=130 -q --w
 
 step ssh into VM
 
-if $systemd_run -d --wait -q -p StandardOutput=journal -- ssh -i ./mkosi.key \
-        -o UserKnownHostsFile=/dev/null \
-        -o StrictHostKeyChecking=no \
-        -o LogLevel=ERROR \
-        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p" \
-        root@mkosi 'mangosctl --base-url=http://10.0.2.2:8081 updatectl add-overrides ; /usr/share/mangos/self_test.sh'
-then
+# Stream the remote self-test live to the workflow console and also save to a logfile
+diag_ssh_out="${tmpdir}/self_test_ssh.out"
+echo "Streaming remote self-test output to ${diag_ssh_out}"
+
+# Use direct ssh (with forced tty) so output is streamed live. Save output with tee.
+# Run ssh+tee in background and tail the logfile in foreground so CI logs show live output
+ssh_cmd=(ssh -tt -i ./mkosi.key
+        -o UserKnownHostsFile=/dev/null
+        -o StrictHostKeyChecking=no
+        -o LogLevel=ERROR
+        -o ProxyCommand="mkosi sandbox -- socat - VSOCK-CONNECT:42:%p"
+        root@mkosi "bash -lc 'mangosctl --base-url=http://10.0.2.2:8081 updatectl add-overrides ; /usr/share/mangos/self_test.sh'")
+
+# Ensure diag file exists
+touch "${diag_ssh_out}"
+
+# Trap to clean child processes on exit
+cleanup_ssh_tail() {
+    if [ -n "${ssh_pid:-}" ]; then
+        kill "${ssh_pid}" 2>/dev/null || true
+    fi
+    if [ -n "${tail_pid:-}" ]; then
+        kill "${tail_pid}" 2>/dev/null || true
+    fi
+}
+trap cleanup_ssh_tail EXIT
+
+# Start ssh pipeline in background, using stdbuf to avoid buffering
+stdbuf -oL "${ssh_cmd[@]}" 2>&1 | stdbuf -oL tee "${diag_ssh_out}" &
+ssh_pid=$!
+
+# Give ssh/tee a moment to start writing, then tail the logfile to stream live output
+sleep 1
+tail -n +1 -f "${diag_ssh_out}" &
+tail_pid=$!
+
+# Wait for ssh to finish
+wait ${ssh_pid}
+ssh_rc=$?
+
+# Stop tailing
+kill ${tail_pid} 2>/dev/null || true
+wait ${tail_pid} 2>/dev/null || true
+
+trap - EXIT
+
+if [ ${ssh_rc} -eq 0 ]; then
     success
-    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} succeeded"
+    echo "Mangos test ${testid} succeeded" | $systemd_run -q -u "mangos-test-${testid}-result" -- cat
 else
     failure
-    $systemd_run -u "mangos-test-${testid}-result" -q -- echo "Mangos test ${testid} failed"
-    exit 1
+    echo "Mangos test ${testid} failed" | $systemd_run -q -u "mangos-test-${testid}-result" -- cat
+    echo "--- Tail of remote self-test output (last 200 lines) ---"
+    tail -n 200 "${diag_ssh_out}" || true
+    exit ${ssh_rc}
 fi