ci: utilize ssh for qemu command execution, replacing the screen-based workflow

Author: ivila

ci/ci.sh

@@ -25,10 +25,12 @@ pushd ../tests
 # probability of detecting failures early in the pipeline.
 # Run std only tests
 if [ "$STD" ]; then
-    ./test_serde.sh
-    ./test_message_passing_interface.sh
+    # move the EXPAND_TA_MEMORY tests together
     ./test_tls_client.sh
     ./test_tls_server.sh
+
+    ./test_serde.sh
+    ./test_message_passing_interface.sh
     ./test_eth_wallet.sh
     ./test_secure_db_abstraction.sh
 else

tests/optee-qemuv8.sh

@@ -32,5 +32,5 @@ cd $1 && ./qemu-system-aarch64 \
     -kernel Image \
     -fsdev local,id=fsdev0,path=$(pwd)/../shared,security_model=none \
     -device virtio-9p-device,fsdev=fsdev0,mount_tag=host \
-    -netdev user,id=vmnic,hostfwd=:127.0.0.1:54433-:4433 \
+    -netdev user,id=vmnic,hostfwd=:127.0.0.1:54433-:4433,hostfwd=:127.0.0.1:54432-:22 \
     -device virtio-net-device,netdev=vmnic

tests/setup.sh

@@ -26,27 +26,73 @@ OPTEE_TAG=optee-$(cat ../optee-version.txt)
 # Define IMG_VERSION
 IMG_VERSION="$(uname -m)-$OPTEE_TAG-qemuv8-ubuntu-24.04"
 
-# Set IMG based on NEED_EXPANDED_MEM
+IMG="$IMG_VERSION"
+NORMAL_SESSION_NAME="qemu_screen"
+EXPAND_MEMORY_SESSION_NAME="qemu_screen_expand_ta_memory"
+
+CURRENT_SESSION_NAME=$NORMAL_SESSION_NAME
+OTHER_SESSION_NAME=$EXPAND_MEMORY_SESSION_NAME
+# Change Options based on NEED_EXPANDED_MEM
 if [ "$NEED_EXPANDED_MEM" = true ]; then
     IMG="${IMG_VERSION}-expand-ta-memory"
-else
-    IMG="$IMG_VERSION"
+    CURRENT_SESSION_NAME=$EXPAND_MEMORY_SESSION_NAME
+    OTHER_SESSION_NAME=$NORMAL_SESSION_NAME
 fi
 
+SSH_PORT=54432
+# StrictHostKeyChecking=no: Bypasses the interactive prompt to confirm the
+#   host's authenticity.
+# UserKnownHostsFile=/dev/null: Prevents saving host keys to disk; this avoids
+#   "Host key verification failed" errors when the QEMU instance restarts with
+#   a new identity.
+SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes"
+SSH_TARGET="root@127.0.0.1"
+
+SCREEN_LOG_PATH=screenlog.0
+SERIAL_LOG_PATH=/tmp/serial.log
+
 # Function to download image
 download_image() {
     curl "https://nightlies.apache.org/teaclave/teaclave-trustzone-sdk/${IMG}.tar.gz" | tar zxv
 }
 
-# Functions for running commands in QEMU screen
+# Functions for running commands in QEMU
 run_in_qemu() {
-    (screen -S qemu_screen -p 0 -X stuff "$1\n") || (echo "run_in_qemu '$1' failed" && cat /tmp/serial.log)
-    sleep 5
+    run_in_qemu_with_timeout_secs "$1" 10s
 }
 
 run_in_qemu_with_timeout_secs() {
-    (screen -S qemu_screen -p 0 -X stuff "$1\n") || (echo "run_in_qemu '$1' failed" && cat /tmp/serial.log)
-    sleep $2
+    timeout "$2" \
+        ssh $SSH_TARGET -p $SSH_PORT $SSH_OPTIONS "$1"
+}
+
+copy_to_qemu() {
+    local dest_path=$1
+    shift
+
+    timeout 60s \
+        scp -P $SSH_PORT $SSH_OPTIONS $@ $SSH_TARGET:"$dest_path"
+}
+
+copy_ta_to_qemu() {
+    copy_to_qemu "/lib/optee_armtz/" $@
+    run_in_qemu "chmod 0444 /lib/optee_armtz/*.ta"
+}
+
+copy_ca_to_qemu() {
+    copy_to_qemu "/usr/bin/" $@
+}
+
+copy_plugin_to_qemu() {
+    copy_to_qemu "/usr/lib/tee-supplicant/plugins/" $@
+    run_in_qemu "chmod 0666 /usr/lib/tee-supplicant/plugins/*.so"
+}
+
+# Functions for handling failure
+print_detail_and_exit() {
+    cat -v $SCREEN_LOG_PATH
+    cat -v $SERIAL_LOG_PATH
+    exit 1
 }
 
 # Check if the image file exists locally
@@ -58,11 +104,29 @@ else
 fi
 
 mkdir -p shared
+# Keeps the shared folder for ease of manual developer verification.
+# "mkdir -p shared && mount -t 9p -o trans=virtio host shared"
 
+# Terminate existing QEMU screen sessions to prevent conflicts.
+if screen -list | grep -q "\.${OTHER_SESSION_NAME}[[:space:]]"; then
+    echo "Other Session '${OTHER_SESSION_NAME}' is running, terminate it to prevent conflicts"
+    screen -S $OTHER_SESSION_NAME -X quit
+    rm -f $SERIAL_LOG_PATH && rm -f $SCREEN_LOG_PATH
+fi
 # Start QEMU screen
-screen -L -d -m -S qemu_screen ./optee-qemuv8.sh $IMG
-sleep 30
-run_in_qemu "root"
-run_in_qemu "mkdir -p shared && mount -t 9p -o trans=virtio host shared && cd shared"
-# libteec.so.2 since OP-TEE 4.2.0, for legacy versions:
-run_in_qemu "[ ! -e /usr/lib/libteec.so.1 ] && ln -s /usr/lib/libteec.so /usr/lib/libteec.so.1"
+if screen -list | grep -q "\.${CURRENT_SESSION_NAME}[[:space:]]"; then
+    echo "Session '${CURRENT_SESSION_NAME}' is already running. Skipping start."
+else
+    echo "Starting new session: ${CURRENT_SESSION_NAME}"
+    screen -L -d -m -S $CURRENT_SESSION_NAME ./optee-qemuv8.sh $IMG
+fi
+
+TEST_QEMU_SCRIPT_NAME=/tmp/teaclave-$CURRENT_SESSION_NAME.sh
+cat <<EOF > "$TEST_QEMU_SCRIPT_NAME"
+    until ssh -p $SSH_PORT $SSH_TARGET $SSH_OPTIONS "true" >/dev/null 2>&1; do
+        printf "."
+        sleep 1
+    done
+EOF
+timeout 30s bash $TEST_QEMU_SCRIPT_NAME
+echo "QEMU SSH Ready"

tests/test_acipher.sh

@@ -23,22 +23,14 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/acipher-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/acipher-rs/host/target/$TARGET_HOST/release/acipher-rs shared
+copy_ta_to_qemu ../examples/acipher-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/acipher-rs/host/target/$TARGET_HOST/release/acipher-rs
 
 # Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
-run_in_qemu "./acipher-rs 256 teststring\n"
-run_in_qemu "^C"
+OUTPUT=$(run_in_qemu "acipher-rs 256 teststring") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "Success encrypt input text \".*\" as [0-9]* bytes cipher text:" screenlog.0 &&
-    grep -q "Success decrypt the above ciphertext as [0-9]* bytes plain text:" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-    false
-}
-
-rm screenlog.0
+    grep -q "Success encrypt input text \".*\" as [0-9]* bytes cipher text:" <<< "$OUTPUT" &&
+    grep -q "Success decrypt the above ciphertext as [0-9]* bytes plain text:" <<< "$OUTPUT"
+} || print_detail_and_exit

tests/test_aes.sh

@@ -23,26 +23,18 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/aes-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/aes-rs/host/target/$TARGET_HOST/release/aes-rs shared
+copy_ta_to_qemu ../examples/aes-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/aes-rs/host/target/$TARGET_HOST/release/aes-rs
 
 # Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
-run_in_qemu "./aes-rs\n"
-run_in_qemu "^C"
+OUTPUT=$(run_in_qemu "aes-rs") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "Prepare encode operation" screenlog.0 &&
-    grep -q "Load key in TA" screenlog.0 &&
-    grep -q "Reset ciphering operation in TA (provides the initial vector)" screenlog.0 &&
-    grep -q "Encode buffer from TA" screenlog.0 &&
-    grep -q "Prepare decode operation" screenlog.0 &&
-    grep -q "Clear text and decoded text match" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-    false
-}
-
-rm screenlog.0
+    grep -q "Prepare encode operation" <<< "$OUTPUT" &&
+    grep -q "Load key in TA" <<< "$OUTPUT" &&
+    grep -q "Reset ciphering operation in TA (provides the initial vector)" <<< "$OUTPUT" &&
+    grep -q "Encode buffer from TA" <<< "$OUTPUT" &&
+    grep -q "Prepare decode operation" <<< "$OUTPUT" &&
+    grep -q "Clear text and decoded text match" <<< "$OUTPUT"
+} || print_detail_and_exit

tests/test_authentication.sh

@@ -23,22 +23,14 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/authentication-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/authentication-rs/host/target/$TARGET_HOST/release/authentication-rs shared
+copy_ta_to_qemu ../examples/authentication-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/authentication-rs/host/target/$TARGET_HOST/release/authentication-rs
 
 # Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
-run_in_qemu "./authentication-rs\n"
-run_in_qemu "^C"
+OUTPUT=$(run_in_qemu "authentication-rs") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "Clear text and decoded text match" screenlog.0 &&
-    grep -q "Success" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-        false
-}
-
-rm screenlog.0
+    grep -q "Clear text and decoded text match" <<< "$OUTPUT" &&
+    grep -q "Success" <<< "$OUTPUT"
+} || print_detail_and_exit

tests/test_big_int.sh

@@ -23,29 +23,13 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/big_int-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/big_int-rs/host/target/$TARGET_HOST/release/big_int-rs shared
+copy_ta_to_qemu ../examples/big_int-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/big_int-rs/host/target/$TARGET_HOST/release/big_int-rs
 
 # Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
-run_in_qemu "./big_int-rs\n"
-run_in_qemu "^C"
+OUTPUT=$(run_in_qemu "big_int-rs") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "\[.*] > \[.*]\|\[.*] < \[.*]\|\[.*] == \[.*]" /tmp/serial.log &&
-    grep -q "\[.*] in u8 array is \[.*]" /tmp/serial.log &&
-    grep -q "\[.*] in i32 is [0-9]*" /tmp/serial.log &&
-    grep -q "\[.*] + \[.*] = \[.*]" /tmp/serial.log &&
-    grep -q "\[.*] - \[.*] = \[.*]" /tmp/serial.log &&
-    grep -q "\[.*] \* \[.*] = \[.*]" /tmp/serial.log &&
-    grep -q "\[.*] / \[.*] = \[.*]" /tmp/serial.log &&
-    grep -q "\[.*] % \[.*] = \[.*]" /tmp/serial.log &&
-    grep -q "Success" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-        false
-}
-
-rm screenlog.0
+    grep -q "Success" <<< "$OUTPUT"
+} || print_detail_and_exit

tests/test_build_with_optee_utee_sys.sh

@@ -23,26 +23,17 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/build_with_optee_utee_sys-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/build_with_optee_utee_sys-rs/host/target/$TARGET_HOST/release/build_with_optee_utee_sys-rs shared
+copy_ta_to_qemu ../examples/build_with_optee_utee_sys-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/build_with_optee_utee_sys-rs/host/target/$TARGET_HOST/release/build_with_optee_utee_sys-rs
 
-# Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
 # Run command twice, ensure the instance are keeping alive.
-run_in_qemu "./build_with_optee_utee_sys-rs\n"
-run_in_qemu "./build_with_optee_utee_sys-rs\n"
-run_in_qemu "^C"
+OUTPUT1=$(run_in_qemu "build_with_optee_utee_sys-rs") || print_detail_and_exit
+OUTPUT2=$(run_in_qemu "build_with_optee_utee_sys-rs") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "result is: 0" screenlog.0 &&
-    grep -q "result is: 1" screenlog.0 &&
-    grep -q "result is: 2" screenlog.0 &&
-    grep -q "result is: 3" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-    false
-}
-
-rm screenlog.0
+    grep -q "result is: 0" <<< "$OUTPUT1" &&
+    grep -q "result is: 1" <<< "$OUTPUT1" &&
+    grep -q "result is: 2" <<< "$OUTPUT2" &&
+    grep -q "result is: 3" <<< "$OUTPUT2"
+} || print_detail_and_exit

tests/test_client_pool.sh

@@ -23,23 +23,15 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/client_pool-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/client_pool-rs/host/target/$TARGET_HOST/release/client_pool-rs shared
+copy_ta_to_qemu ../examples/client_pool-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/client_pool-rs/host/target/$TARGET_HOST/release/client_pool-rs
 
 # Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
-run_in_qemu "./client_pool-rs thread -p 2 -c 2 -t 500 -e 2000\n"
-run_in_qemu "./client_pool-rs async -p 2 -c 2 -t 500 -e 2000\n"
-run_in_qemu "^C"
+OUTPUT1=$(run_in_qemu "client_pool-rs thread -p 2 -c 2 -t 500 -e 2000") || print_detail_and_exit
+OUTPUT2=$(run_in_qemu "client_pool-rs async -p 2 -c 2 -t 500 -e 2000") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "r2d2: total tasks: 2, total finish: 2" screenlog.0 &&
-    grep -q "mobc: total tasks: 2, total finish: 2" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-    false
-}
-
-rm screenlog.0
+    grep -q "r2d2: total tasks: 2, total finish: " <<< "$OUTPUT1" &&
+    grep -q "mobc: total tasks: 2, total finish: " <<< "$OUTPUT2"
+} || print_detail_and_exit

tests/test_diffie_hellman.sh

@@ -23,23 +23,15 @@ set -xe
 source setup.sh
 
 # Copy TA and host binary
-cp ../examples/diffie_hellman-rs/ta/target/$TARGET_TA/release/*.ta shared
-cp ../examples/diffie_hellman-rs/host/target/$TARGET_HOST/release/diffie_hellman-rs shared
+copy_ta_to_qemu ../examples/diffie_hellman-rs/ta/target/$TARGET_TA/release/*.ta
+copy_ca_to_qemu ../examples/diffie_hellman-rs/host/target/$TARGET_HOST/release/diffie_hellman-rs
 
 # Run script specific commands in QEMU
-run_in_qemu "cp *.ta /lib/optee_armtz/\n"
-run_in_qemu "./diffie_hellman-rs\n"
-run_in_qemu "^C"
+OUTPUT=$(run_in_qemu "diffie_hellman-rs") || print_detail_and_exit
 
 # Script specific checks
 {
-    grep -q "get key [0|1] pair as public: \[.*], private: \[.*]" screenlog.0 &&
-    grep -q "Derived share key as \[.*]" screenlog.0 &&
-    grep -q "Success" screenlog.0
-} || {
-    cat -v screenlog.0
-    cat -v /tmp/serial.log
-        false
-}
-
-rm screenlog.0
+    grep -q "get key [0|1] pair as public: \[.*], private: \[.*]" <<< "$OUTPUT" &&
+    grep -q "Derived share key as \[.*]" <<< "$OUTPUT" &&
+    grep -q "Success" <<< "$OUTPUT"
+} || print_detail_and_exit