diff --git a/misc/AMDSEV/README.md b/misc/AMDSEV/README.md index ce693ca3e..1e17bf731 100644 --- a/misc/AMDSEV/README.md +++ b/misc/AMDSEV/README.md @@ -40,7 +40,7 @@ If `--katana` is not provided, `build.sh` prompts for confirmation (`y/N`) befor | `build-kernel.sh` | Downloads and extracts Ubuntu kernel (`vmlinuz`) | | `build-initrd.sh` | Creates minimal initrd with busybox, SEV-SNP modules, and katana | | `build-config` | Pinned versions and checksums for reproducible builds | -| `start-vm.sh` | Starts a TEE VM with SEV-SNP enabled using QEMU | +| `start-vm.sh` | Starts a TEE VM with SEV-SNP and launches Katana asynchronously | ## SNP Tools @@ -69,6 +69,9 @@ cargo build -p snp-tools ## Running +The QEMU command below boots the VM but does not automatically start Katana. +Katana must be started asynchronously via the control channel. + ```sh qemu-system-x86_64 \ # Use KVM hardware virtualization (required for SEV-SNP) @@ -93,11 +96,42 @@ qemu-system-x86_64 \ # Initial ramdisk containing katana (measured when kernel-hashes=on) -initrd output/qemu/initrd.img \ # Kernel command line (measured when kernel-hashes=on) - # katana.args passes arguments to katana via init script - -append "console=ttyS0 katana.args=--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp" \ + -append "console=ttyS0" \ + # Katana control channel (used to start Katana asynchronously after boot) + -device virtio-serial-pci,id=virtio-serial0 \ + -chardev socket,id=katanactl,path=/tmp/katana-control.sock,server=on,wait=off \ + -device virtserialport,chardev=katanactl,name=org.katana.control.0 \ .. ``` +### Start Katana via Control Channel + +In the QEMU example above, this line defines the host-side control channel endpoint: + +```sh +-chardev socket,id=katanactl,path=/tmp/katana-control.sock,server=on,wait=off +``` + +The `path=/tmp/katana-control.sock` value is the Unix socket file on the host. +That socket is connected to the guest virtio-serial port: + +```sh +-device virtserialport,chardev=katanactl,name=org.katana.control.0 +``` + +So writes to that Unix socket become control commands inside the VM (`start`, `status`). + +Example: + +```sh +# Start Katana with comma-separated CLI args +printf 'start --http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp\n' \ + | socat - UNIX-CONNECT:/tmp/katana-control.sock + +# Check launcher status +printf 'status\n' | socat - UNIX-CONNECT:/tmp/katana-control.sock +``` + ## Running the VM The `start-vm.sh` script provides an easy way to launch a TEE VM with SEV-SNP enabled: @@ -108,11 +142,16 @@ sudo ./misc/AMDSEV/start-vm.sh # Or specify a custom boot components directory sudo ./misc/AMDSEV/start-vm.sh /path/to/boot-components + +# Or customize Katana runtime flags (comma-separated) +sudo ./misc/AMDSEV/start-vm.sh --katana-args "--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp,--dev" ``` The script: - Starts QEMU with SEV-SNP confidential computing enabled - Uses direct kernel boot with kernel-hashes=on for attestation +- Keeps kernel cmdline stable (`console=ttyS0`) for deterministic measurement +- Starts Katana asynchronously via virtio-serial control channel - Forwards RPC port 5050 to host port 15051 - Outputs serial log to a temp file and follows it @@ -129,7 +168,7 @@ cargo build -p snp-tools --ovmf output/qemu/OVMF.fd \ --kernel output/qemu/vmlinuz \ --initrd output/qemu/initrd.img \ - --append "console=ttyS0 katana.args=--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp" \ + --append "console=ttyS0" \ --vcpus 1 \ --cpu epyc-v4 \ --vmm qemu \ diff --git a/misc/AMDSEV/build-initrd.sh b/misc/AMDSEV/build-initrd.sh index 90f9794bf..5abf1ac04 100755 --- a/misc/AMDSEV/build-initrd.sh +++ b/misc/AMDSEV/build-initrd.sh @@ -293,6 +293,9 @@ log() { echo "[init] $*"; } KATANA_PID="" KATANA_DB_DIR="/mnt/data/katana-db" SHUTTING_DOWN=0 +KATANA_EXIT_CODE="never" +CONTROL_PORT_NAME="org.katana.control.0" +CONTROL_PORT_LINK="/dev/virtio-ports/org.katana.control.0" fatal_boot() { log "ERROR: $*" @@ -303,6 +306,90 @@ fatal_boot() { done } +refresh_katana_state() { + if [ -n "$KATANA_PID" ] && ! kill -0 "$KATANA_PID" 2>/dev/null; then + if wait "$KATANA_PID"; then + KATANA_EXIT_CODE=0 + else + KATANA_EXIT_CODE=$? + fi + log "Katana exited with code $KATANA_EXIT_CODE" + KATANA_PID="" + fi +} + +respond_control() { + printf '%s\n' "$1" >&3 2>/dev/null || true +} + +resolve_control_port() { + mkdir -p /dev/virtio-ports + for name_file in /sys/class/virtio-ports/*/name; do + [ -f "$name_file" ] || continue + + PORT_NAME_VALUE="$(cat "$name_file" 2>/dev/null || true)" + if [ "$PORT_NAME_VALUE" != "$CONTROL_PORT_NAME" ]; then + continue + fi + + PORT_DIR="${name_file%/name}" + PORT_DEV="/dev/${PORT_DIR##*/}" + if [ -e "$PORT_DEV" ]; then + ln -sf "$PORT_DEV" "$CONTROL_PORT_LINK" + echo "$CONTROL_PORT_LINK" + return 0 + fi + done + return 1 +} + +handle_control_command() { + RAW_CMD="$1" + CMD="${RAW_CMD%% *}" + CMD_PAYLOAD="" + if [ "$CMD" != "$RAW_CMD" ]; then + CMD_PAYLOAD="${RAW_CMD#* }" + fi + + case "$CMD" in + start) + refresh_katana_state + if [ -n "$KATANA_PID" ] && kill -0 "$KATANA_PID" 2>/dev/null; then + respond_control "err already-running pid=$KATANA_PID" + return 0 + fi + + KATANA_ARGS="" + if [ -n "$CMD_PAYLOAD" ]; then + KATANA_ARGS="$(echo "$CMD_PAYLOAD" | tr ',' ' ')" + fi + + log "Starting katana asynchronously..." + # shellcheck disable=SC2086 + /bin/katana --db-dir="$KATANA_DB_DIR" $KATANA_ARGS & + KATANA_PID=$! + KATANA_EXIT_CODE="running" + respond_control "ok started pid=$KATANA_PID" + ;; + + status) + refresh_katana_state + if [ -n "$KATANA_PID" ] && kill -0 "$KATANA_PID" 2>/dev/null; then + respond_control "running pid=$KATANA_PID" + else + respond_control "stopped exit=$KATANA_EXIT_CODE" + fi + ;; + + "") + ;; + + *) + respond_control "err unknown-command" + ;; + esac +} + shutdown_handler() { if [ "$SHUTTING_DOWN" -eq 1 ]; then return 0 @@ -404,15 +491,6 @@ else log "WARNING: eth0 interface not found; skipping static network setup" fi -# Parse katana args from cmdline -CMDLINE="$(cat /proc/cmdline 2>/dev/null || true)" -KATANA_ARGS="" -for tok in $CMDLINE; do - case "$tok" in - katana.args=*) KATANA_ARGS="$(echo "${tok#katana.args=}" | tr ',' ' ')" ;; - esac -done - # Require persistent storage at /dev/sda if [ ! -b /dev/sda ]; then fatal_boot "required storage device /dev/sda not found" @@ -426,22 +504,30 @@ fi mkdir -p "$KATANA_DB_DIR" log "Storage mounted at /mnt/data" -log "Starting katana..." -# shellcheck disable=SC2086 -/bin/katana --db-dir="$KATANA_DB_DIR" $KATANA_ARGS & -KATANA_PID=$! -log "Katana started with PID $KATANA_PID" - -if wait "$KATANA_PID"; then - EXIT_CODE=0 -else - EXIT_CODE=$? -fi -log "Katana exited with code $EXIT_CODE" +# Start async control loop for Katana startup/status commands. +log "Waiting for control channel ($CONTROL_PORT_NAME)..." +CONTROL_PORT="" +while [ -z "$CONTROL_PORT" ]; do + CONTROL_PORT="$(resolve_control_port || true)" + [ -n "$CONTROL_PORT" ] || sleep 1 +done +log "Control channel ready: $CONTROL_PORT" -# PID 1 must stay alive unless explicitly powered off. while true; do - sleep 60 + refresh_katana_state + + if ! exec 3<>"$CONTROL_PORT"; then + log "WARNING: failed to open control channel, retrying..." + sleep 1 + continue + fi + + while IFS= read -r CONTROL_CMD <&3; do + handle_control_command "$CONTROL_CMD" + done + + exec 3>&- 3<&- + sleep 1 done INIT_EOF diff --git a/misc/AMDSEV/start-vm.sh b/misc/AMDSEV/start-vm.sh index 6ecbdbb75..fc6326d08 100755 --- a/misc/AMDSEV/start-vm.sh +++ b/misc/AMDSEV/start-vm.sh @@ -1,11 +1,12 @@ #!/bin/bash # Start TEE VM with AMD SEV-SNP -# Usage: ./start-vm.sh [BOOT_COMPONENTS_DIR] +# Usage: ./start-vm.sh [BOOT_COMPONENTS_DIR] [--katana-args CSV] [--no-start] # # This script: # 1. Starts QEMU with the TEE boot components -# 2. Runs katana without persistent storage (in-memory only) -# 3. Forwards RPC port to host +# 2. Creates and attaches a data disk as /dev/sda +# 3. Optionally starts Katana asynchronously via a virtio-serial control channel +# 4. Forwards RPC port to host # # ============================================================================== # LAUNCH MEASUREMENT INPUTS @@ -14,33 +15,86 @@ # measurement. Verifiers must use the same values to reproduce the measurement. # # Boot components (hashed when kernel-hashes=on): -# OVMF_FILE - OVMF.fd firmware image -# KERNEL_FILE - vmlinuz kernel image -# INITRD_FILE - initrd.img initial ramdisk -# KERNEL_CMDLINE - "console=ttyS0 katana.args=--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp" +# OVMF_FILE - OVMF.fd firmware image +# KERNEL_FILE - vmlinuz kernel image +# INITRD_FILE - initrd.img initial ramdisk +# KERNEL_CMDLINE - "console=ttyS0" # # SEV-SNP guest configuration: -# GUEST_POLICY - 0x30000 (SMT allowed, debug disabled) -# VCPU_COUNT - 1 -# VMSA_FEATURES - 0x1 (SNP active) +# GUEST_POLICY - 0x30000 (SMT allowed, debug disabled) +# VCPU_COUNT - 1 +# GUEST_FEATURES - 0x1 (SNP active) # # CPU and platform: -# CPU_TYPE - EPYC-v4 -# CBITPOS - 51 (C-bit position for memory encryption) +# CPU_TYPE - EPYC-v4 +# CBITPOS - 51 (C-bit position for memory encryption) # REDUCED_PHYS_BITS - 1 # +# Katana launch arguments are sent after boot over a control channel and are NOT +# part of the measured kernel command line. +# # To compute expected measurement, use snp-digest from snp-tools: # cargo build -p snp-tools # ./target/debug/snp-digest --ovmf=OVMF.fd --kernel=vmlinuz --initrd=initrd.img \ -# --append="console=ttyS0 katana.args=--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp" \ -# --vcpus=1 --cpu=epyc-v4 --vmm=qemu --guest-features=0x1 +# --append="console=ttyS0" --vcpus=1 --cpu=epyc-v4 --vmm=qemu --guest-features=0x1 # # ============================================================================== set -euo pipefail +usage() { + echo "Usage: $0 [BOOT_COMPONENTS_DIR] [--katana-args CSV] [--no-start]" + echo "" + echo "Starts a SEV-SNP VM and launches Katana asynchronously via control channel." + echo "" + echo "Arguments:" + echo " BOOT_COMPONENTS_DIR Optional path containing OVMF.fd, vmlinuz, initrd.img" + echo "" + echo "Options:" + echo " --katana-args CSV Comma-separated Katana CLI args sent after boot" + echo " --no-start Boot VM without sending Katana start command" + echo " -h, --help Show this help" +} + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BOOT_DIR="${1:-$SCRIPT_DIR/output/qemu}" +BOOT_DIR="${SCRIPT_DIR}/output/qemu" +KATANA_ARGS_CSV="--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp" +AUTO_START_KATANA=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + --katana-args) + [[ $# -ge 2 ]] || { + echo "Error: --katana-args requires a value" + exit 1 + } + KATANA_ARGS_CSV="$2" + shift 2 + ;; + + --no-start) + AUTO_START_KATANA=0 + shift + ;; + + -h|--help) + usage + exit 0 + ;; + + -*) + echo "Error: Unknown option: $1" + echo "" + usage + exit 1 + ;; + + *) + BOOT_DIR="$1" + shift + ;; + esac +done # ------------------------------------------------------------------------------ # Launch measurement inputs (must match values documented above) @@ -50,7 +104,7 @@ BOOT_DIR="${1:-$SCRIPT_DIR/output/qemu}" OVMF_FILE="$BOOT_DIR/OVMF.fd" KERNEL_FILE="$BOOT_DIR/vmlinuz" INITRD_FILE="$BOOT_DIR/initrd.img" -KERNEL_CMDLINE="console=ttyS0 katana.args=--http.addr,0.0.0.0,--http.port,5050,--tee.provider,sev-snp" +KERNEL_CMDLINE="console=ttyS0" # SEV-SNP guest configuration GUEST_POLICY="0x30000" @@ -66,8 +120,32 @@ CPU_TYPE="EPYC-v4" KATANA_RPC_PORT=5050 HOST_RPC_PORT=15051 -# Create random temp file for serial log -SERIAL_LOG=$(mktemp /tmp/katana-tee-vm-serial.XXXXXX.log) +# Katana control channel +CONTROL_PORT_NAME="org.katana.control.0" +CONTROL_SOCKET="/tmp/katana-tee-vm-control.$$.sock" +CONTROL_TIMEOUT=60 + +# VM data disk (required by init script) +DISK_IMAGE="$(mktemp /tmp/katana-tee-vm-data.XXXXXX.img)" +DISK_SIZE_MB=1024 + +# Logs +SERIAL_LOG="$(mktemp /tmp/katana-tee-vm-serial.XXXXXX.log)" + +show_serial_tail() { + echo "" + echo "=== Serial output (last 80 lines) ===" + tail -80 "$SERIAL_LOG" 2>/dev/null || echo "(no output)" +} + +send_control_command() { + local cmd="$1" + local response + + response="$(printf '%s\n' "$cmd" | socat -t 2 -T 2 - UNIX-CONNECT:"$CONTROL_SOCKET" 2>/dev/null | head -n1 || true)" + [[ -n "$response" ]] || return 1 + echo "$response" +} # Cleanup function QEMU_PID="" @@ -77,17 +155,15 @@ cleanup() { echo "" echo "=== Cleanup ===" - if [ -n "$QEMU_PID" ] && kill -0 "$QEMU_PID" 2>/dev/null; then + if [[ -n "$QEMU_PID" ]] && kill -0 "$QEMU_PID" 2>/dev/null; then echo "Stopping QEMU (PID $QEMU_PID)..." kill "$QEMU_PID" 2>/dev/null || true - # Wait for QEMU to fully terminate - for i in $(seq 1 10); do + for _ in $(seq 1 10); do if ! kill -0 "$QEMU_PID" 2>/dev/null; then break fi sleep 0.5 done - # Force kill if still running if kill -0 "$QEMU_PID" 2>/dev/null; then echo "Force killing QEMU..." kill -9 "$QEMU_PID" 2>/dev/null || true @@ -95,44 +171,66 @@ cleanup() { wait "$QEMU_PID" 2>/dev/null || true fi - # Clean up serial log file - if [ -f "$SERIAL_LOG" ]; then - rm -f "$SERIAL_LOG" - fi + [[ -f "$SERIAL_LOG" ]] && rm -f "$SERIAL_LOG" + [[ -S "$CONTROL_SOCKET" ]] && rm -f "$CONTROL_SOCKET" + [[ -f "$DISK_IMAGE" ]] && rm -f "$DISK_IMAGE" echo "=== Cleanup complete ===" - exit $exit_code + exit "$exit_code" } trap cleanup EXIT INT TERM -# Check for root/sudo (needed for KVM access) -if [ "$EUID" -ne 0 ]; then - echo "This script requires root privileges for KVM access." +# Check for root/sudo (needed for KVM and disk formatting) +if [[ "$EUID" -ne 0 ]]; then + echo "This script requires root privileges for KVM and disk setup." echo "Please run with: sudo $0 $*" exit 1 fi +for cmd in qemu-system-x86_64 mkfs.ext4 dd; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "Error: Required command not found: $cmd" + exit 1 + fi +done + +if [[ "$AUTO_START_KATANA" -eq 1 ]]; then + if ! command -v socat >/dev/null 2>&1; then + echo "Error: Required command not found: socat" + echo "Install socat or run with --no-start." + exit 1 + fi +fi + # Verify files exist echo "Checking TEE boot components..." for file in "$OVMF_FILE" "$KERNEL_FILE" "$INITRD_FILE"; do - if [ ! -f "$file" ]; then + if [[ ! -f "$file" ]]; then echo "Error: Missing $file" exit 1 fi echo " Found: $file ($(ls -lh "$file" | awk '{print $5}'))" done +# Prepare required ext4 data disk for /dev/sda +echo "" +echo "Preparing VM data disk..." +dd if=/dev/zero of="$DISK_IMAGE" bs=1M count="$DISK_SIZE_MB" status=none +mkfs.ext4 -F -q "$DISK_IMAGE" +echo " Disk image: $DISK_IMAGE (${DISK_SIZE_MB}MB, ext4)" + echo "" echo "Starting TEE QEMU VM..." -echo " OVMF: $OVMF_FILE" -echo " Kernel: $KERNEL_FILE" -echo " Initrd: $INITRD_FILE" -echo " Cmdline: $KERNEL_CMDLINE" -echo " Policy: $GUEST_POLICY" -echo " vCPUs: $VCPU_COUNT" -echo " Memory: $MEMORY" -echo " Serial: $SERIAL_LOG" -echo " RPC: localhost:$HOST_RPC_PORT -> VM:$KATANA_RPC_PORT" +echo " OVMF: $OVMF_FILE" +echo " Kernel: $KERNEL_FILE" +echo " Initrd: $INITRD_FILE" +echo " Cmdline: $KERNEL_CMDLINE" +echo " Policy: $GUEST_POLICY" +echo " vCPUs: $VCPU_COUNT" +echo " Memory: $MEMORY" +echo " Serial: $SERIAL_LOG" +echo " Control socket: $CONTROL_SOCKET" +echo " RPC: localhost:$HOST_RPC_PORT -> VM:$KATANA_RPC_PORT" echo "" echo "To compute expected launch measurement:" echo " snp-digest --ovmf=$OVMF_FILE --kernel=$KERNEL_FILE --initrd=$INITRD_FILE \\" @@ -153,6 +251,12 @@ qemu-system-x86_64 \ -kernel "$KERNEL_FILE" \ -initrd "$INITRD_FILE" \ -append "$KERNEL_CMDLINE" \ + -device virtio-serial-pci,id=virtio-serial0 \ + -chardev socket,id=katanactl,path="$CONTROL_SOCKET",server=on,wait=off \ + -device virtserialport,chardev=katanactl,name="$CONTROL_PORT_NAME" \ + -device virtio-scsi-pci,id=scsi0 \ + -drive file="$DISK_IMAGE",format=raw,if=none,id=disk0,cache=none \ + -device scsi-hd,drive=disk0,bus=scsi0.0 \ -netdev user,id=net0,hostfwd=tcp::${HOST_RPC_PORT}-:${KATANA_RPC_PORT} \ -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=net0,romfile= \ & @@ -163,15 +267,90 @@ echo "QEMU started with PID $QEMU_PID" # Wait for serial log file to be created echo "" echo "Waiting for serial log file..." -while [ ! -f "$SERIAL_LOG" ]; do +while [[ ! -f "$SERIAL_LOG" ]]; do if ! kill -0 "$QEMU_PID" 2>/dev/null; then echo "Error: QEMU process died before creating serial log" + show_serial_tail exit 1 fi sleep 0.1 done echo "Serial log file created" +if [[ "$AUTO_START_KATANA" -eq 1 ]]; then + echo "" + echo "Waiting for control socket..." + waited=0 + while [[ ! -S "$CONTROL_SOCKET" ]]; do + if ! kill -0 "$QEMU_PID" 2>/dev/null; then + echo "Error: QEMU process died before control socket became ready" + show_serial_tail + exit 1 + fi + + sleep 1 + waited=$((waited + 1)) + if [[ "$waited" -ge "$CONTROL_TIMEOUT" ]]; then + echo "Error: Timeout waiting for control socket: $CONTROL_SOCKET" + show_serial_tail + exit 1 + fi + done + echo "Control socket ready" + + echo "" + echo "Sending async Katana start command..." + START_RESPONSE="$(send_control_command "start $KATANA_ARGS_CSV" || true)" + if [[ -z "$START_RESPONSE" ]]; then + echo "Error: No response from guest control channel" + show_serial_tail + exit 1 + fi + echo " Start response: $START_RESPONSE" + + case "$START_RESPONSE" in + ok\ started*|err\ already-running*) + ;; + *) + echo "Error: Unexpected start response from guest" + show_serial_tail + exit 1 + ;; + esac + + echo "" + echo "Waiting for Katana running status..." + waited=0 + while true; do + if ! kill -0 "$QEMU_PID" 2>/dev/null; then + echo "Error: QEMU process died while waiting for Katana" + show_serial_tail + exit 1 + fi + + STATUS_RESPONSE="$(send_control_command "status" || true)" + if [[ "$STATUS_RESPONSE" == running\ * ]]; then + echo " Status: $STATUS_RESPONSE" + break + fi + + sleep 1 + waited=$((waited + 1)) + if [[ "$waited" -ge "$CONTROL_TIMEOUT" ]]; then + echo "Error: Timeout waiting for Katana to report running" + echo " Last status: ${STATUS_RESPONSE:-}" + show_serial_tail + exit 1 + fi + done +else + echo "" + echo "Katana auto-start disabled (--no-start)." + echo "Use the control socket to send commands manually:" + echo " printf 'start $KATANA_ARGS_CSV\n' | socat - UNIX-CONNECT:$CONTROL_SOCKET" + echo " printf 'status\n' | socat - UNIX-CONNECT:$CONTROL_SOCKET" +fi + echo "" echo "=== Following serial output (Ctrl+C to exit) ===" tail -f "$SERIAL_LOG"