feat: Enable CEX based LUKS encryption

    1. Configure the CEX in KVM host
    2. Use the right device type ignition for MCO
    3. Attach the CEX mediated device to guest vm

Signed-off-by: Madhu Pillai <[email protected]>

Author: madhu-pillai

docs/set-variables-group-vars.md

@@ -12,6 +12,7 @@
 :--- | :--- | :---
 **installation_type** | Can be of type kvm or lpar. Some packages will be ignored for installation in case of non lpar based installation. | kvm 
 **controller_sudo_pass** | The password to the machine running Ansible (localhost). This will only be used for two things. To ensure you've installed the pre-requisite packages if you're on Linux, and to add the login URL to your /etc/hosts file. | Pas$w0rd!
+**cex_device** | Specify the storage device type used for LUKS encryption. This setting determines enable cex MCO Ignition configuration will be applied. Use in combination with the cex parameter.  [dasd, fcp, virt]
 
 ## 2 - LPAR(s)
 **Variable Name** | **Description** | **Example**
@@ -403,3 +404,7 @@
 **zvm.interface.ip** | IP addresses for to be used for zVM nodes | 192.168.10.1
 **zvm.nodes.dasd.disk_id** | Disk id for dasd disk to be used for zVM node | 4404
 **zvm.nodes.lun** | Disk details of fcp disk to be used for zVM node | 840a
+
+## Crypto Express Card based LUKS encryption specific for zKVM ( Optional )
+**Variable Name** | **Description** | **Example**
+**cex_uuid_map** | Specify guest hostname: "UUID:domain" UUID can be generated from uuidgen command and domain can be retrieved from lszcrypt | upi-cex-control-1: "68cd2d83-3eef-4e45-b22c-534f90b16cb9:00.0035"

inventories/default/group_vars/all.yaml.template

@@ -6,6 +6,7 @@
 # Section 1 - Ansible Controller
 installation_type: kvm
 controller_sudo_pass: #X
+cex_device: #[dasd | fcp | virt ]
 
 env:
 
@@ -250,3 +251,13 @@ abi:
 # (Optional) Proxy
 # Pls check the documentation which vars are present (include examples). If use_proxy is set to true,
 # than proxy_http, proxy_https and proxy_no must be set.
+
+
+# Section 15 - CEX based Luks Encryption ( Optional )
+cex_uuid_map:
+  hostname:UUID:domain
+  hostname:UUID:domain
+
+# Provide control and compute hostname with uuid and domain.
+# UUID can be generated from `uuidgen` command and domain from lszcrypt -V
+# eg upi-control-1:5c84eefb-cb45-4519-86d3-ba23e65e8896:12.0001

inventories/default/hosts

@@ -188,3 +188,9 @@
   roles:
     - configure_storage
     - { role: macvtap, when: env.network_mode | upper != 'NAT' }
+
+- hosts: kvm_host
+  tags: setup, section_3
+  become: true
+  roles:
+    - { role: configure_cex, when: cex_device is defined }

playbooks/3_setup_kvm_host.yaml

@@ -0,0 +1,40 @@
+---
+
+- name: Set cex_cards from cex_uuid_map values (uuid:domain only)
+  set_fact:
+    cex_cards: "{{ cex_uuid_map.values() | list | unique }}"
+
+- name: Debug final list of CEX UUID assignments
+  debug:
+    var: cex_cards
+
+- name: Create VFIO assignment script for all CEX cards
+  template:
+    src: assign_cards.sh.j2
+    dest: /tmp/assign_all_cex_cards.sh
+    mode: '0755'
+
+- name: Execute VFIO assignment script
+  shell: /tmp/assign_all_cex_cards.sh
+  args:
+    executable: /bin/bash
+
+- name: Housekeep temporary assignment script
+  file:
+    path: /tmp/assign_all_cex_cards.sh
+    state: absent
+
+- name: Initialize empty cex_hostdev_map
+  set_fact:
+    cex_hostdev_map: {}
+
+- name: Populate cex_hostdev_map with mdev format
+  set_fact:
+    cex_hostdev_map: "{{ cex_hostdev_map | combine({ item.key : 'mdev_' + (item.value.split(':')[0] | regex_replace('-', '_')) + '_matrix' }) }}"
+  loop: "{{ cex_uuid_map | dict2items }}"
+
+
+- name: Save cex_hostdev_map to a file for reuse
+  copy:
+    dest: "/root/.cex_hostdev_map.json"
+    content: "{{ cex_hostdev_map | to_nice_json }}"

roles/configure_cex/tasks/main.yaml

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Reference document for the cex configuration in zKVM
+# https://www.ibm.com/docs/en/linux-on-systems?topic=management-configuring-crypto-express-adapters-kvm-guests
+
+# Configure each CEX card
+{% for entry in cex_cards %}
+{% set uuid = entry.split(':')[0] %}
+{% set matrix_val = entry.split(':')[1] %}
+{% set adapter = matrix_val.split('.')[0] %}
+{% set domain = matrix_val.split('.')[1] %}
+
+uuid="{{ uuid }}"
+matrix_val="{{ matrix_val }}"
+adapter="{{ adapter }}"
+domain="{{ domain }}"
+
+uuid_path="/sys/devices/vfio_ap/matrix/$uuid"
+matrix_file="$uuid_path/matrix"
+
+if [ -d "$uuid_path" ]; then
+    if grep -q "{{ matrix_val }}" "$matrix_file" 2>/dev/null; then
+        echo "[INFO] UUID $uuid already configured with matrix {{ matrix_val }} — skipping."
+    else
+        echo "[WARN] UUID $uuid exists, but matrix entry '{{ matrix_val }}' not found!"
+        echo "[WARN] Please reboot the node and try again."
+        exit 1
+    fi
+else
+    modprobe vfio_ap
+    echo 0x0 > /sys/bus/ap/apmask
+    echo 0x0 > /sys/bus/ap/aqmask
+    echo "[INFO] Creating UUID $uuid with adapter $adapter and domain $domain"
+    echo "$uuid" > /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/create
+    echo "0x$adapter" > "$uuid_path/assign_adapter"
+    echo "0x$domain" > "$uuid_path/assign_domain"
+fi
+
+{% endfor %}

roles/configure_cex/templates/assign_cards.sh.j2

@@ -1,4 +1,7 @@
 ---
+- name: Load cex_hostdev_map from JSON
+  set_fact:
+    cex_hostdev_map: "{{ lookup('file', '/root/.cex_hostdev_map.json') | from_json }}"
 
 - name: Install CoreOS on compute nodes
   tags: create_compute_nodes
@@ -30,7 +33,10 @@
     --memballoon none \
     --graphics none \
     --wait=-1 \
-    --noautoconsole
+    --noautoconsole \
+    {% set vm_name = env.cluster.nodes.compute.vm_name[i] %}
+    {% set hostdev = '--hostdev ' + cex_hostdev_map[vm_name] if cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map else '' %}
+    {{ hostdev }}
   timeout: 360
   with_sequence: start=0 end={{ (env.cluster.nodes.compute.hostname | length) - 1 }} stride=1
   loop_control:

roles/create_compute_nodes/tasks/main.yaml

@@ -1,7 +1,32 @@
 ---
 
+- name: Load cex_hostdev_map from JSON file
+  set_fact:
+    cex_hostdev_map: "{{ lookup('file', '/root/.cex_hostdev_map.json') | from_json }}"
+
+- name: Debug rendered cex_hostdev per control node
+  debug:
+    msg: "VM: {{ vm_name }}, Hostdev: {{ cex_hostdev }}"
+  vars:
+    vm_name: "{{ env.cluster.nodes.control.vm_name[i] }}"
+    cex_hostdev: >-
+      {% if cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map %}
+      --hostdev={{ cex_hostdev_map[vm_name] }}
+      {% else %}
+      ""
+      {% endif %}
+  with_sequence: start=0 end={{ (env.cluster.nodes.control.hostname | length) - 1 }}
+  loop_control:
+    index_var: i
+
 - name: Create CoreOS control nodes on the the KVM host.
   tags: create_control_nodes
+  vars:
+    vm_name: "{{ env.cluster.nodes.control.vm_name[i] }}"
+    cex_hostdev: >-
+      {% if cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map %}
+      --hostdev={{ cex_hostdev_map[vm_name] }}
+      {% endif %}
   shell: |
     virt-install \
     --name {{ env.cluster.nodes.control.vm_name[i] }} \
@@ -29,7 +54,8 @@
     --graphics none \
     --console pty,target_type=serial \
     --wait=-1 \
-    --noautoconsole
+    --noautoconsole \
+    {{ cex_hostdev }}
   timeout: 360
   with_sequence: start=0 end={{ (env.cluster.nodes.control.hostname | length) - 1 }} stride=1
   loop_control:
@@ -67,6 +93,7 @@
     --graphics none \
     --wait=-1 \
     --noautoconsole
+
   when: env.z.high_availability == True and inventory_hostname == env.z.lpar1.hostname and env.cluster.nodes.control.vm_name[0] not in hosts_with_host_vars
 
 - name: Create the second CoreOS control node on the second KVM host, if cluster is to be highly available.

roles/create_control_nodes/tasks/main.yaml

@@ -0,0 +1,16 @@
+variant: openshift
+version: 4.19.0
+metadata:
+  name: master-luks-storage
+  labels:
+    machineconfiguration.openshift.io/role: master
+openshift:
+  fips: true
+  kernel_arguments:
+    - rd.luks.key=/etc/luks/cex.key
+boot_device:
+  layout: s390x-eckd
+  luks:
+    device: /dev/dasda
+    cex:
+      enabled: true

roles/get_ocp/files/dasd-master-machineconfig.bu

@@ -0,0 +1,30 @@
+# Generated by Butane; do not edit
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: master
+  name: master-luks-storage
+spec:
+  config:
+    ignition:
+      version: 3.5.0
+    storage:
+      filesystems:
+        - device: /dev/mapper/root
+          format: xfs
+          label: root
+          wipeFilesystem: true
+      luks:
+        - cex:
+            enabled: true
+          device: /dev/dasda2
+          label: luks-root
+          name: root
+          options:
+            - --cipher
+            - aes-cbc-essiv:sha256
+          wipeVolume: true
+  fips: true
+  kernelArguments:
+    - rd.luks.key=/etc/luks/cex.key