Set up CEX device on s390x builder

Add a new systemd unit that sets up a CEX device for use by kola tests.
The device is associated with a UUID. This UUID is passed to kola via
an environment variable for now.

See also: coreos/coreos-assembler#3828

Co-authored-by: Jonathan Lebon <[email protected]>

Author: madhu-pillai

config.yaml

@@ -23,6 +23,12 @@ streams:
 
 additional_arches: [aarch64, ppc64le, s390x]
 
+env:
+  # This matches the UUID in multi-arch-builders/create-cex-device.sh
+  # Ideally, we'd pass this a cleaner way. See also:
+  # https://github.com/coreos/coreos-assembler/pull/3828#discussion_r1669010741
+  KOLA_CEX_UUID: "68cd2d83-3eef-4e45-b22c-534f90b16cb9"
+
 source_config:
   url: https://github.com/coreos/fedora-coreos-config
 

multi-arch-builders/coreos-s390x-rhcos-builder.bu

@@ -29,6 +29,17 @@ systemd:
         ExecStart=/usr/local/bin/create-secex-data.sh
         [Install]
         WantedBy=multi-user.target
+    - name: cex-config.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Create IBM CEX mediate device
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/local/bin/create-cex-device.sh
+        [Install]
+        WantedBy=multi-user.target
 storage:
   files:
     - path: /usr/local/bin/create-secex-data.sh
@@ -91,3 +102,12 @@ storage:
 
             echo "Importing tarball into volume"
             sudo -u builder -H /bin/bash -c "cd /var/home/builder; podman volume import secex-data /var/home/builder/${TARBALL}"
+
+    - path: /usr/local/bin/create-cex-device.sh
+      mode: 0755
+      user:
+        name: root
+      group:
+        name: root
+      contents:
+        local: create-cex-device.sh

multi-arch-builders/create-cex-device.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -euo pipefail
+
+# Reference document for the cex configuration in zKVM
+# https://www.ibm.com/docs/en/linux-on-systems?topic=management-configuring-crypto-express-adapters-kvm-guests
+
+modprobe vfio_ap
+
+# Hardcoded the UUID for the quick verification of mediated device.
+# This UUID is passed to kola via the KOLA_CEX_UUID env var in the pipecfg.
+UUID='68cd2d83-3eef-4e45-b22c-534f90b16cb9'
+if  ls /sys/devices/vfio_ap/matrix | grep -q "${UUID}"; then
+    echo "${UUID}"
+    exit 1
+fi
+
+cards=$(ls /sys/bus/ap/devices | grep -E 'card[0-9]{2}')
+if [ -z "${cards}" ]; then
+    echo "cex device not found."
+    exit 1
+fi
+
+# if more than one card picks the first available cca controller
+for card in $cards; do
+    if $(grep -q cca /sys/bus/ap/devices/${card}/uevent); then
+        card=${card}
+        card_domain=$(ls /sys/bus/ap/devices/$card/ | grep -E '[0-9a-f]{2}.[0-9a-f]{4}')
+        break
+    fi
+done
+
+# Validating the card and domain
+if [ -z "${card}" ] || [ -z "${card_domain}"]; then
+    echo "couldn't find card with CCA controller"
+    exit 1
+fi
+
+echo "Freeing adapter and domain."
+echo 0x0 > /sys/bus/ap/apmask
+echo 0x0 >  /sys/bus/ap/aqmask
+echo ${UUID} > /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/create
+if [ $? != 0 ]; then
+    echo "failed creating mediated device."
+    exit 1
+fi
+
+echo "Configuring the adapter and domain."
+card_no=$(echo ${card_domain} | cut -f1 -d.)
+domain_no=$(echo ${card_domain} | cut -f2 -d.)
+echo 0x${card_no} > /sys/devices/vfio_ap/matrix/${UUID}/assign_adapter
+echo 0x${domain_no} > /sys/devices/vfio_ap/matrix/${UUID}/assign_domain
+
+echo "Validating the domains."
+validate_dom=$(cat /sys/devices/vfio_ap/matrix/${UUID}/matrix)
+if [[ "${card_no}.${domain_no}" != ${validate_dom} ]]; then
+    echo -e "Mismatched card number. \n"
+    exit 1
+fi