[CI] clean up the qemu workflow and scripts a little

- rename 'qemu_config.py' to 'parse_config.py' (it's already in qemu dir);
- minor changes, removal of redundant code, extra comments, etc.

Author: lukaszstolarczuk

.github/workflows/qemu.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-    - name: Checkout
+    - name: Checkout UMF
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         fetch-depth: 0
@@ -25,16 +25,20 @@ jobs:
         echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
         sudo udevadm control --reload-rules
         sudo udevadm trigger --name-match=kvm
-    - name: Install qemu
-      run: |
-        sudo apt update && sudo apt install -y qemu-system genisoimage qemu-utils
-    - name: Install libvirt and script dependencies
+
+    - name: Install dependencies
       run: |
-        sudo apt-get install -y libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils
+        sudo apt-get update
+        sudo apt-get install -y qemu-system genisoimage qemu-utils \
+          libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils
         pip install -r scripts/qemu/requirements.txt
-        sudo usermod -a -G kvm,libvirt $USER
+
+    - name: Add user to kvm group
+      run: sudo usermod -a -G kvm,libvirt $USER
+
     - name: Run ssh-keygen
       run: ssh-keygen -b 4096 -N '' -f ~/.ssh/id_rsa
+
     - name: Generate iso with user info
       run: |
         pub_key=$(cat ~/.ssh/id_rsa.pub)
@@ -69,11 +73,14 @@ jobs:
         EOF
 
         sudo -Sk genisoimage -output ubuntu-cloud-init.iso -volid cidata -joliet -rock ./user-data ./meta-data
+
     - name: Download ubuntu image
       run: wget https://cloud-images.ubuntu.com/releases/lunar/release/ubuntu-23.04-server-cloudimg-amd64.img
+
     - name: Resize image
       run: qemu-img resize ./ubuntu-23.04-server-cloudimg-amd64.img +4G
-    - name: Build
+
+    - name: Build UMF in QEMU
       run: |
         scripts/qemu/start_qemu.sh scripts/qemu/configs/default.xml
 
@@ -85,27 +92,22 @@ jobs:
 
         scp -P 2222 ${{github.workspace}}/scripts/qemu/run-build.sh [email protected]:/home/cxltest
         scp -P 2222 ${{github.workspace}}/scripts/qemu/run-tests.sh [email protected]:/home/cxltest
-        ssh [email protected] -p 2222 -t "bash /home/cxltest/run-build.sh https://github.com/$CI_REPO ${{env.CI_BRANCH}}"
 
+        ssh [email protected] -p 2222 -t "bash /home/cxltest/run-build.sh https://github.com/$CI_REPO ${{env.CI_BRANCH}}"
         ssh [email protected] -p 2222 -t "sudo shutdown -h now"
 
-    - name: Run tests
+    - name: Run tests in QEMU
       run: |
         for config_file in scripts/qemu/configs/*.xml; do
           config_name=$(basename $config_file .xml)
 
-          echo testing $config_name
           while ps -aux | grep qemu-system-x86_64 | grep -q -v grep; do
             echo "Waiting for QEMU to shut down..."
             sleep 5
           done
+
+          echo "\n ### Testing ${config_name} ###"
           scripts/qemu/start_qemu.sh $config_file
-          
-          if [ ${{ github.event_name }} = 'pull_request' ]; then
-            CI_REPO="${{ github.event.pull_request.head.repo.full_name }}"
-          else
-            CI_REPO="$GITHUB_REPOSITORY"
-          fi
 
           ssh [email protected] -p 2222 -t "bash /home/cxltest/run-tests.sh"
           ssh [email protected] -p 2222 -t "sudo shutdown -h now"

scripts/qemu/qemu_config.py renamed to scripts/qemu/parse_config.py

@@ -13,10 +13,13 @@
 import psutil
 import shutil
 
-# If you want to manually run this script please install deps by: pip install -r requirements.txt
-# To get virsh please install libvirt-clients
+# This script parses the topology xml file and returns QEMU arguments.
 #
-# Enable verbose mode by using environment variable ENABLE_VERBOSE=1
+# Before running this script:
+# - install python deps for this script: pip install -r requirements.txt
+# - install 'libvirt-clients' package (for virsh)
+#
+# Enable verbose mode by setting environment variable: ENABLE_VERBOSE=1
 
 TopologyCfg = collections.namedtuple(
     "TopologyCfg", ["name", "hmat", "cpu_model", "cpu_options", "mem_options"]
@@ -27,7 +30,7 @@
 
 def enable_verbose():
     """
-    Parse command line arguments
+    Check if env var ENABLE_VERBOSE is set and enable verbose mode
     """
     global verbose_mode
     verbose_mode = os.getenv("ENABLE_VERBOSE", False)
@@ -50,6 +53,9 @@ def parse_topology_xml(tpg_file_name: str) -> TopologyCfg:
         result.check_returncode()
         libvirt_args = result.stdout.decode("utf-8").strip()
 
+        if verbose_mode != False:
+            print(f"\nFull libvirt_args: {libvirt_args}\n")
+
         tpg_cfg = {
             "name": re.search(r"guest=(\w+)", libvirt_args).group(1),
             "hmat": "hmat=on" in libvirt_args,
@@ -74,7 +80,7 @@ def parse_topology_xml(tpg_file_name: str) -> TopologyCfg:
     except subprocess.CalledProcessError:
         sys.exit(f"\n XML file: {tpg_file_name} error in virsh parsing")
     except Exception:
-        sys.exit(f"\n Provided file is missing or missing virsh.")
+        sys.exit(f"\n Provided file ({tpg_file_name}) is missing or missing virsh.")
     return tpg
 
 
@@ -89,7 +95,7 @@ def get_qemu_args(tpg_file_name: str) -> str:
 
 def calculate_memory(tpg: TopologyCfg) -> str:
     """
-    Memory used by QEMU
+    Total memory required by given QEMU config
     """
     if tpg.mem_options:
         mem_needed = 0
@@ -112,4 +118,6 @@ def calculate_memory(tpg: TopologyCfg) -> str:
         tpg_file_name = sys.argv[1]
     else:
         sys.exit(f"\n Usage: {sys.argv[0]} <tpg_file_name>")
+
+    # Print QEMU arguments as a result of this script
     print(get_qemu_args(tpg_file_name))

scripts/qemu/run-build.sh

@@ -8,8 +8,9 @@ set -e
 repo=$1
 branch=$2
 
-echo password | sudo -Sk apt update
-echo password | sudo -Sk apt install -y git cmake gcc g++ numactl libnuma-dev libhwloc-dev libjemalloc-dev libtbb-dev pkg-config valgrind hwloc
+echo password | sudo -Sk apt-get update
+echo password | sudo -Sk apt-get install -y git cmake gcc g++ pkg-config \
+    numactl libnuma-dev hwloc libhwloc-dev libjemalloc-dev libtbb-dev valgrind
 
 git clone $repo umf
 cd umf

scripts/qemu/run-tests.sh

@@ -23,4 +23,3 @@ numactl -N 1 ctest --output-on-failure
 # run tests under valgrind
 echo "Running tests under valgrind memcheck ..."
 ../test/test_valgrind.sh .. . memcheck
-

scripts/qemu/start_qemu.sh

@@ -3,12 +3,12 @@
 # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-set -x
 set -e
 
 config_file=$1
 
-python3 scripts/qemu/qemu_config.py $config_file
+# Parse the config file to get topology info and fix escaped single quotes
+parsed_config=$(python3 scripts/qemu/parse_config.py ${config_file} | sed s/''\''/'/g)
 
 if grep -q '<interconnects>' "$config_file"; then
     hmat="on"
@@ -19,12 +19,13 @@ fi
 sudo qemu-system-x86_64 \
     -drive file=./ubuntu-23.04-server-cloudimg-amd64.img,format=qcow2,index=0,media=disk,id=hd \
     -cdrom ./ubuntu-cloud-init.iso \
-    -machine q35,usb=off,hmat=$hmat \
+    -machine q35,usb=off,hmat=${hmat} \
     -enable-kvm \
     -net nic -net user,hostfwd=tcp::2222-:22 \
-    $(python3 scripts/qemu/qemu_config.py $config_file | sed s/''\''/'/g) \
+    ${parsed_config} \
     -daemonize -display none
 
+# Enable ssh connection to the VM
 until ssh-keyscan -p 2222 -H 127.0.0.1 >> ~/.ssh/known_hosts 2>/dev/null; do
     echo "Waiting for SSH..."
     sleep 1