WIP: add lustre role

Author: sjpb

ansible/.gitignore

@@ -58,4 +58,5 @@ roles/*
 !roles/squid/**
 !roles/tuned/
 !roles/tuned/**
-
+!roles/lustre/
+!roles/lustre/**

ansible/fatimage.yml

@@ -44,6 +44,11 @@
         name: stackhpc.os-manila-mount
         tasks_from: install.yml
       when: "'manila' in group_names"
+    - name: Install Lustre packages
+      include_role:
+        name: lustre
+        tasks_from: install.yml
+      when: "'lustre' in group_names"
 
 - import_playbook: extras.yml
 

ansible/filesystems.yml

@@ -24,3 +24,12 @@
   tasks:
     - include_role:
         name: stackhpc.os-manila-mount
+
+- name: Setup Lustre clients
+  hosts: lustre
+  become: true
+  tags: lustre
+  tasks:
+    - include_role:
+        name: lustre
+        tasks_from: configure.yml

ansible/roles/lustre/README.md

@@ -0,0 +1,23 @@
+# lustre
+
+Install and configure a Lustre client. This builds RPM packages from source.
+
+**NB:** The `install.yml` playbook in this role should only be run during image build, with the default `update_enable=true`. This ensures that the latest kernel and matching
+`kernel-devel` packages will be installed. This playbook is not idempotent.
+
+## Role Variables
+
+- `lustre_version`: Optional str. Version of lustre to build, default '2.15.64'. TODO: EXPLAIN. See https://wiki.whamcloud.com/display/PUB/Lustre+Support+Matrix
+- `lustre_mounts`: Optional list. Define Lustre filesystems and mountpoints as a list of dicts with possible keys:
+    - `lustre_mgs_nid`: The NID for the MGS, e.g. `192.168.227.11@tcp1`
+    - `lustre_fs_name`: The name of the filesystem to mount
+    - `lustre_mount_point`: Path to mount filesystem at. Default is `/mnt/lustre/{{ lustre_fs_name}}`
+    - `lustre_mount_state`: Mountpoint state, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default `mounted`.
+  These parameters may alternatively be specified as role variables of the same name, which are lower-priority than entries in `lustre_mounts`. 
+
+The following variables control the package build and and install and should not generally be required:
+- `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`.
+- `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`.
+- `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellnox OFED, without support for GSS keys. See `defaults/main.yml`.
+- `lustre_rpm_globs`: Optional list. Shell glob patterns for rpms to install. Note order is important as the built RPMs are not in a yum repo. Default is just the `kmod-lustre-client` and `lustre-client` packages.
+- `lustre_cleanup_build`: Optional bool. Whether to uninstall prerequisite packages and delete the build directories etc. Default `true`.

ansible/roles/lustre/defaults/main.yml

@@ -0,0 +1,35 @@
+
+lustre_version: 'b2_15' # TODO: FIXME to 2.15.5 on release see http://lists.lustre.org/pipermail/lustre-discuss-lustre.org/2024-June/019163.html
+lustre_mounts: []
+lustre_mount_point: /mnt/lustre
+lustre_mount_state: mounted
+
+# below variables are for build and should not generally require changes
+lustre_build_packages:
+  - "kernel-devel-{{ ansible_kernel }}"
+  - git
+  - gcc
+  - libtool
+  - python3
+  - python3-devel
+  - openmpi
+  - elfutils-libelf-devel
+  - libmount-devel
+  - libnl3-devel
+  - libyaml-devel
+  - rpm-build
+  - kernel-abi-stablelists
+  - libaio
+  - libaio-devel
+lustre_build_dir: /tmp/lustre-release
+lustre_configure_opts:
+  - --disable-server
+  - --with-linux=/usr/src/kernels/*
+  - --with-o2ib=/usr/src/ofa_kernel/default
+  - --disable-maintainer-mode
+  - --disable-gss-keyring
+  - --enable-mpitests=no
+lustre_rpm_globs: # NB: order is important here, as not installing from a repo
+  - "kmod-lustre-client-{{ lustre_version }}-*"
+  - "lustre-client-{{ lustre_version }}-*"
+lustre_cleanup_build: true

ansible/roles/lustre/tasks/configure.yml

@@ -0,0 +1,9 @@
+# TODO: is this an ok way of doing it?
+- name: Mount lustre filesystem
+  mount:
+    fstype: lustre
+    src: "{{ lustre_mgs_nid }}:/{{ lustre_fs_name }}"
+    path: "{{ lustre_mount_point }}"
+    state: "{{ lustre_mount_state }}"
+    opts: "defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service"
+    # opts are systemd defaults from http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes

ansible/roles/lustre/tasks/install.yml

@@ -0,0 +1,56 @@
+- name: Install lustre build prerequisites
+  ansible.builtin.dnf:
+    name: "{{ lustre_build_packages }}"
+  register: _lustre_dnf_build_packages
+ 
+- name: Clone lustre git repo
+  ansible.builtin.git:
+    repo: git://git.whamcloud.com/fs/lustre-release.git
+    dest: "{{ lustre_build_dir }}"
+    version: "{{ lustre_version }}"
+
+- name: Prepare for lustre configuration
+  ansible.builtin.command:
+    cmd: ./autogen.sh
+    chdir: "{{ lustre_build_dir }}"
+
+- name: Configure lustre build
+  ansible.builtin.command:
+    cmd: "./configure {{ lustre_configure_opts | join(' ') }}"
+    chdir: "{{ lustre_build_dir }}"
+
+- name: Build lustre
+  ansible.builtin.command:
+    cmd: make rpms
+    chdir: "{{ lustre_build_dir }}"
+
+- name: Find rpms
+  ansible.builtin.find:
+    paths: "{{ lustre_build_dir }}"
+    patterns: "{{ lustre_rpm_globs }}"
+    use_regex: false
+  register: _lustre_find_rpms
+
+- name: Check rpms found
+  assert:
+    that: _lustre_find_rpms.files | length
+    fail_msg: "No lustre repos found with lustre_rpm_globs = {{ lustre_rpm_globs }}"
+
+- name: Install lustre rpms
+  ansible.builtin.dnf:
+    name: "{{ _lustre_find_rpms.files | map(attribute='path')}}"
+    disable_gpg_check: yes
+  
+- block:
+  - name: Remove lustre build prerequisites
+    # NB only remove ones this role installed!
+    ansible.builtin.dnf:
+      name: "{{ item | regex_replace('Installed: ', '') }}"
+      state: absent
+    loop: "{{ _lustre_dnf_build_packages.results }}"
+  
+  - name: Delete lustre build dir
+    file:
+      path: "{{ lustre_build_dir }}"
+      state: absent
+  when: lustre_cleanup_build | bool

ansible/roles/lustre/tasks/validate.yml

@@ -0,0 +1,26 @@
+- name: Assert using RockyLinux 9
+  assert:
+    that: ansible_distribution_major_version | int == 9
+    fail_msg: The 'lustre' role requires RockyLinux 9
+
+- name: Check kernel-devel package is installed
+  command: "dnf list --installed kernel-devel-{{ ansible_kernel }}"
+  changed_when: false
+  # NB: we don't check here the kernel will remain the same after reboot etc, see ofed/install.yml
+
+- name: Ensure SELinux in permissive mode
+  assert:
+    that: selinux_state in ['permissive', 'disabled']
+    fail_msg: "SELinux must be permissive for Lustre not '{{ selinux_state }}'; see variable selinux_state"
+
+- name: Ensure lustre_mgs_nid is defined
+  assert:
+    that: item.lustre_mgs_nid | default(lustre_mgs_nid) is defined
+    fail_msg: lustre_mgs_nid must be defined or specified in lustre_mounts elements
+  loop: "{{ lustre_mounts }}"
+
+- name: Ensure lustre_fs_name is defined
+  assert:
+    that: item.lustre_fs_name | default(lustre_fs_name) is defined
+    fail_msg: lustre_fs_name must be defined or specified in lustre_mounts elements
+  loop: "{{ lustre_mounts }}"

ansible/validate.yml

@@ -95,3 +95,11 @@
     - import_role:
         name: freeipa
         tasks_from: validate.yml
+
+- name: Validate lustre configuration
+  hosts: lustre
+  tags: lustre
+  tasks:
+    - import_role:
+        name: lustre
+        tasks_from: validate.yml

environments/common/inventory/groups

@@ -134,4 +134,7 @@ freeipa_client
 # Hosts to run TuneD configuration
 
 [ansible_init]
-# Hosts to run linux-anisble-init
+# Hosts to run linux-anisble-init
+
+[lustre]
+# Hosts to run lustre client