diff --git a/docs/set-variables-group-vars.md b/docs/set-variables-group-vars.md index 197fb1cf1..2a774af1d 100644 --- a/docs/set-variables-group-vars.md +++ b/docs/set-variables-group-vars.md @@ -12,6 +12,7 @@ :--- | :--- | :--- **installation_type** | Can be of type kvm or lpar. Some packages will be ignored for installation in case of non lpar based installation. | kvm **controller_sudo_pass** | The password to the machine running Ansible (localhost). This will only be used for two things. To ensure you've installed the pre-requisite packages if you're on Linux, and to add the login URL to your /etc/hosts file. | Pas$w0rd! +**cex_device** | Specify the storage device type used for LUKS encryption. This setting determines enable cex MCO Ignition configuration will be applied. Use in combination with the cex parameter. [dasd, fcp, virt] ## 2 - LPAR(s) **Variable Name** | **Description** | **Example** @@ -409,3 +410,9 @@ **zvm.interface.ip** | IP addresses for to be used for zVM nodes | 192.168.10.1 **zvm.nodes.dasd.disk_id** | Disk id for dasd disk to be used for zVM node | 4404 **zvm.nodes.lun** | Disk details of fcp disk to be used for zVM node | 840a + +## Crypto Express Card based LUKS encryption specific for zKVM ( Optional ) +**Variable Name** | **Description** | **Example** +**cex** | Whether to enable cex based luks encryption, default to False +**cex_device** | Specify the storage device type used for LUKS encryption. This setting determines which MCO Ignition configuration will be applied from the defaults. Do not override the default value. Use in combination with the cex parameter. | [dasd, fcp, virt] +**cex_uuid_map** | This var is required only for KVM installations using vfio_ap mediated device. Omit it when deploying on LPAR installation. Use in combination with cex and cex_device. Specify guest hostname: "UUID:domain" UUID can be generated from uuidgen command and domain can be retrieved from lszcrypt | upi-cex-control-1: "68cd2d83-3eef-4e45-b22c-534f90b16cb9:00.0035" \ No newline at end of file diff --git a/inventories/default/group_vars/all.yaml.template b/inventories/default/group_vars/all.yaml.template index e9d191a8c..a4da48984 100644 --- a/inventories/default/group_vars/all.yaml.template +++ b/inventories/default/group_vars/all.yaml.template @@ -250,3 +250,16 @@ abi: # (Optional) Proxy # Pls check the documentation which vars are present (include examples). If use_proxy is set to true, # than proxy_http, proxy_https and proxy_no must be set. + + +# Section 15 - CEX based Luks Encryption ( Optional ) +cex: false +# cex_device: [dasd | fcp | virt] +# The following variable is required only when CEX use as vfio_ap mediate device in KVM guest. +# https://www.ibm.com/docs/en/linux-on-systems?topic=management-configuring-crypto-express-adapters-kvm-guests +# cex_uuid_map: +# hostname:UUID:domain +# hostname:UUID:domain +# Provide control and compute hostname with uuid and domain only for KVM based installation. +# UUID can be generated from `uuidgen` command and domain from lszcrypt -V +# eg upi-control-1:5c84eefb-cb45-4519-86d3-ba23e65e8896:12.0001 diff --git a/inventories/default/hosts b/inventories/default/hosts index 45b5d232e..eceebfe3d 100644 --- a/inventories/default/hosts +++ b/inventories/default/hosts @@ -1,2 +1,2 @@ [localhost] -127.0.0.1 ansible_connection=local +127.0.0.1 ansible_connection=local ansible_become_password= diff --git a/playbooks/3_setup_kvm_host.yaml b/playbooks/3_setup_kvm_host.yaml index 97bb855b9..7b7d78d45 100644 --- a/playbooks/3_setup_kvm_host.yaml +++ b/playbooks/3_setup_kvm_host.yaml @@ -188,3 +188,9 @@ roles: - configure_storage - { role: macvtap, when: env.network_mode | upper != 'NAT' } + +- hosts: kvm_host + tags: setup, section_3 + become: true + roles: + - { role: configure_cex, when: cex | bool and cex_uuid_map is defined } diff --git a/roles/configure_cex/tasks/main.yaml b/roles/configure_cex/tasks/main.yaml new file mode 100644 index 000000000..45a955bf3 --- /dev/null +++ b/roles/configure_cex/tasks/main.yaml @@ -0,0 +1,40 @@ +--- + +- name: Set cex_cards from cex_uuid_map values (uuid:domain only) + set_fact: + cex_cards: "{{ cex_uuid_map.values() | list | unique }}" + +- name: Debug final list of CEX UUID assignments + debug: + var: cex_cards + +- name: Create VFIO assignment script for all CEX cards + template: + src: assign_cards.sh.j2 + dest: /tmp/assign_all_cex_cards.sh + mode: '0755' + +- name: Execute VFIO assignment script + shell: /tmp/assign_all_cex_cards.sh + args: + executable: /bin/bash + +- name: Housekeep temporary assignment script + file: + path: /tmp/assign_all_cex_cards.sh + state: absent + +- name: Initialize empty cex_hostdev_map + set_fact: + cex_hostdev_map: {} + +- name: Populate cex_hostdev_map with mdev format + set_fact: + cex_hostdev_map: "{{ cex_hostdev_map | combine({ item.key : 'mdev_' + (item.value.split(':')[0] | regex_replace('-', '_')) + '_matrix' }) }}" + loop: "{{ cex_uuid_map | dict2items }}" + + +- name: Save cex_hostdev_map to a file for reuse + copy: + dest: "/root/.cex_hostdev_map.json" + content: "{{ cex_hostdev_map | to_nice_json }}" diff --git a/roles/configure_cex/templates/assign_cards.sh.j2 b/roles/configure_cex/templates/assign_cards.sh.j2 new file mode 100644 index 000000000..5071213a5 --- /dev/null +++ b/roles/configure_cex/templates/assign_cards.sh.j2 @@ -0,0 +1,38 @@ +#!/bin/bash +# Reference document for the cex configuration in zKVM +# https://www.ibm.com/docs/en/linux-on-systems?topic=management-configuring-crypto-express-adapters-kvm-guests + +# Configure each CEX card +{% for entry in cex_cards %} +{% set uuid = entry.split(':')[0] %} +{% set matrix_val = entry.split(':')[1] %} +{% set adapter = matrix_val.split('.')[0] %} +{% set domain = matrix_val.split('.')[1] %} + +uuid="{{ uuid }}" +matrix_val="{{ matrix_val }}" +adapter="{{ adapter }}" +domain="{{ domain }}" + +uuid_path="/sys/devices/vfio_ap/matrix/$uuid" +matrix_file="$uuid_path/matrix" + +if [ -d "$uuid_path" ]; then + if grep -q "{{ matrix_val }}" "$matrix_file" 2>/dev/null; then + echo "[INFO] UUID $uuid already configured with matrix {{ matrix_val }} — skipping." + else + echo "[WARN] UUID $uuid exists, but matrix entry '{{ matrix_val }}' not found!" + echo "[WARN] Please reboot the node and try again." + exit 1 + fi +else + modprobe vfio_ap + echo 0x0 > /sys/bus/ap/apmask + echo 0x0 > /sys/bus/ap/aqmask + echo "[INFO] Creating UUID $uuid with adapter $adapter and domain $domain" + echo "$uuid" > /sys/devices/vfio_ap/matrix/mdev_supported_types/vfio_ap-passthrough/create + echo "0x$adapter" > "$uuid_path/assign_adapter" + echo "0x$domain" > "$uuid_path/assign_domain" +fi + +{% endfor %} diff --git a/roles/create_compute_nodes/tasks/main.yaml b/roles/create_compute_nodes/tasks/main.yaml index ab8fab9d0..2e2c9ef05 100644 --- a/roles/create_compute_nodes/tasks/main.yaml +++ b/roles/create_compute_nodes/tasks/main.yaml @@ -1,4 +1,23 @@ --- +- name: Load cex_hostdev_map from JSON + set_fact: + cex_hostdev_map: "{{ lookup('file', '/root/.cex_hostdev_map.json') | from_json }}" + when: cex_uuid_map is defined + +- name: Debug rendered cex_hostdev per compute node + debug: + msg: "VM: {{ vm_name }}, Hostdev: {{ cex_hostdev }}" + vars: + vm_name: "{{ env.cluster.nodes.compute.vm_name[i] }}" + cex_hostdev: >- + {% if cex and cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map %} + --hostdev={{ cex_hostdev_map[vm_name] }} + {% else %} + "" + {% endif %} + with_sequence: start=0 end={{ (env.cluster.nodes.compute.hostname | length) - 1 }} + loop_control: + index_var: i - name: 'Include matching lpar yml file' tags: create_teuthology_node @@ -7,6 +26,12 @@ - name: Install CoreOS on compute nodes tags: create_compute_nodes + vars: + vm_name: "{{ env.cluster.nodes.compute.vm_name[i] }}" + cex_hostdev: >- + {% if cex and cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map %} + --hostdev={{ cex_hostdev_map[vm_name] }} + {% endif %} shell: | virsh destroy {{ env.cluster.nodes.compute.vm_name[i] }} || true virsh undefine {{ env.cluster.nodes.compute.vm_name[i] }} --remove-all-storage --nvram || true @@ -35,7 +60,8 @@ --memballoon none \ --graphics none \ --wait=-1 \ - --noautoconsole + --noautoconsole \ + {{ cex_hostdev }} timeout: 360 with_sequence: start=0 end={{ (env.cluster.nodes.compute.hostname | length) - 1 }} stride=1 loop_control: diff --git a/roles/create_control_nodes/tasks/main.yaml b/roles/create_control_nodes/tasks/main.yaml index ef55f0107..1d30e7792 100644 --- a/roles/create_control_nodes/tasks/main.yaml +++ b/roles/create_control_nodes/tasks/main.yaml @@ -5,8 +5,34 @@ ansible.builtin.include_vars: file: "{{ inventory_dir }}/host_vars/{{ inventory_hostname }}.yaml" +- name: Load cex_hostdev_map from JSON file + set_fact: + cex_hostdev_map: "{{ lookup('file', '/root/.cex_hostdev_map.json') | from_json }}" + when: cex_uuid_map is defined + +- name: Debug rendered cex_hostdev per control node + debug: + msg: "VM: {{ vm_name }}, Hostdev: {{ cex_hostdev }}" + vars: + vm_name: "{{ env.cluster.nodes.control.vm_name[i] }}" + cex_hostdev: >- + {% if cex and cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map %} + --hostdev={{ cex_hostdev_map[vm_name] }} + {% else %} + "" + {% endif %} + with_sequence: start=0 end={{ (env.cluster.nodes.control.hostname | length) - 1 }} + loop_control: + index_var: i + - name: Create CoreOS control nodes on the the KVM host. tags: create_control_nodes + vars: + vm_name: "{{ env.cluster.nodes.control.vm_name[i] }}" + cex_hostdev: >- + {% if cex and cex_device is defined and cex_hostdev_map is defined and vm_name in cex_hostdev_map %} + --hostdev={{ cex_hostdev_map[vm_name] }} + {% endif %} shell: | virt-install \ --name {{ env.cluster.nodes.control.vm_name[i] }} \ @@ -34,7 +60,8 @@ --graphics none \ --console pty,target_type=serial \ --wait=-1 \ - --noautoconsole + --noautoconsole \ + {{ cex_hostdev }} timeout: 360 with_sequence: start=0 end={{ (env.cluster.nodes.control.hostname | length) - 1 }} stride=1 loop_control: @@ -72,6 +99,7 @@ --graphics none \ --wait=-1 \ --noautoconsole + when: env.z.high_availability == True and inventory_hostname == env.z.lpar1.hostname and env.cluster.nodes.control.vm_name[0] not in hosts_with_host_vars - name: Create the second CoreOS control node on the first KVM host, if cluster is to be highly available. diff --git a/roles/get_ocp/defaults/main.yaml b/roles/get_ocp/defaults/main.yaml index 4b9307ee8..736841975 100644 --- a/roles/get_ocp/defaults/main.yaml +++ b/roles/get_ocp/defaults/main.yaml @@ -32,3 +32,18 @@ use_proxy: false proxy_http: proxy_https: proxy_no: + +# (Optional) CEX Ignition specific +# Default mappings based on cex_device +output_dir: "/tmp" +butane_default: + version: 4.19.0 + dasd: + layout: s390x-eckd + device: /dev/dasd + virt: + layout: s390x-virt + device: /dev/disk/by-partlabel/root + fcp: + layout: s390x-fcp # or s390x-fcp if needed + device: /dev/disk/by-label/root diff --git a/roles/get_ocp/tasks/main.yaml b/roles/get_ocp/tasks/main.yaml index 530666799..ab1586a0c 100644 --- a/roles/get_ocp/tasks/main.yaml +++ b/roles/get_ocp/tasks/main.yaml @@ -129,6 +129,49 @@ - .openshift_install.log - .openshift_install_state.json +- name: Generate Butane Ignition configs if CEX device is defined + tags: get_ocp + become: true + block: + - name: Generate Butane file for nodes + template: + src: cex-butane-machineconfig.bu.j2 + dest: "{{ output_dir }}/99-{{ node_role }}-s390x-cex-luks-config.bu" + loop: + - master + - worker + loop_control: + loop_var: node_role + vars: + luks_device: "{{ butane_default[cex_device].device }}" + layout: "{{ butane_default[cex_device].layout }}" + + - name: Download Butane binary for s390x + get_url: + url: https://mirror.openshift.com/pub/openshift-v4/clients/butane/latest/butane-s390x + dest: /usr/local/bin/butane + mode: '0755' + + - name: Convert Butane YAML to Ignition for each role + shell: | + /usr/local/bin/butane "{{ output_dir }}/99-{{ item }}-s390x-cex-luks-config.bu" \ + -o "{{ output_dir }}/99-{{ item }}-s390x-cex-luks-config.yaml" + loop: + - master + - worker + + - name: Copy generated ignition files to final directory + copy: + src: "{{ output_dir }}/99-{{ item }}-s390x-cex-luks-config.yaml" + dest: "/root/ocpinst/openshift/99-{{ item }}-s390x-cex-luks-config.yaml" + remote_src: yes + mode: '0644' + loop: + - master + - worker + when: + - cex | bool + - name: Set ownership of ocpinst directory contents to root tags: get_ocp become: true diff --git a/roles/get_ocp/templates/cex-butane-machineconfig.bu.j2 b/roles/get_ocp/templates/cex-butane-machineconfig.bu.j2 new file mode 100644 index 000000000..236f0b145 --- /dev/null +++ b/roles/get_ocp/templates/cex-butane-machineconfig.bu.j2 @@ -0,0 +1,35 @@ +variant: openshift +version: {{ butane_default["version"] }} +metadata: + name: {{ node_role }}-luks-storage + labels: + machineconfiguration.openshift.io/role: {{ node_role }} + +openshift: + fips: true + kernel_arguments: + - rd.luks.key=/etc/luks/cex.key + +{% if cex_device in ['dasd', 'virt'] %} +boot_device: + layout: {{ butane_default[cex_device].layout }} + luks: + device: {{ butane_default[cex_device].device }} + cex: + enabled: true + +{% elif cex_device == 'fcp' %} +storage: + filesystems: + - device: /dev/mapper/root + format: xfs + label: root + wipe_filesystem: true + luks: + - cex: + enabled: true + device: {{ butane_default[cex_device].device }} + label: luks-root + name: root + wipe_volume: true +{% endif %}