refactor lustre role for multiple mounts, selectable lnet label

Author: sjpb

ansible/roles/lustre/README.md

@@ -2,23 +2,26 @@
 
 Install and configure a Lustre client. This builds RPM packages from source.
 
-**NB:** The `install.yml` playbook in this role should only be run during image build, with the default `update_enable=true`. This ensures that the latest kernel and matching
-`kernel-devel` packages will be installed. This playbook is not idempotent.
+**NB:** The `install.yml` playbook in this role should only be run during image build and is not idempotent. This will install the `kernel-devel` package; if not already installed (e.g. from an `ofed` installation), this may require enabling update of DNF packages during build using `update_enable=true`, which will upgrade the kernel as well.
 
 **NB:** Currently this only supports RockyLinux 9.
 
 ## Role Variables
 
-- `lustre_version`: Optional str. Version of lustre to build, default '2.15.5' which is the first version with EL9 support
-- `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with possible keys:
-    - `mgs_nid`: The NID for the MGS, e.g. `192.168.227.11@tcp1`
-    - `fs_name`: The name of the filesystem to mount
-    - `mount_point`: Path to mount filesystem at. Default is `/mnt/lustre/{{ lustre_fs_name}}`
-    - `mount_state`: Mountpoint state, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default `mounted`.
-  TODO: FIXME: Any of these parameters may alternatively be specified as role variables prefixed `lustre_`. If both are given entries in `lustre_mounts` take priority.
+- `lustre_version`: Optional str. Version of lustre to build, default `2.15.5` which is the first version with EL9 support
+- `lustre_lnet_label`: Optional str. The "lnet label" part of the host's NID, e.g. `tcp0` or `o2ib1`. Default `tcp`.
+- `lustre_mgs_nid`: Required str. The NID(s) for the MGS, e.g. `192.168.227.11@tcp1` (separate mutiple MGS NIDs using `:`).
+- `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with keys:
+    - `fs_name`: Required str. The name of the filesystem to mount
+    - `mount_point`: Required str. Path to mount filesystem at.
+    - `mount_state`: Optional mount state, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `lustre_mount_state`.
+    - `mount_options`: Optional mount options. Default is `lustre_mount_options`.
+- `lustre_mount_state`. Optional default mount state for all mounts, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `mounted`.
+- `lustre_mount_options`. Optional default mount options. Default values are systemd defaults from [Lustre client docs](http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes).
+
 The following variables control the package build and and install and should not generally be required:
 - `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`.
 - `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`.
-- `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellnox OFED, without support for GSS keys. See `defaults/main.yml`.
+- `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellanox OFED, without support for GSS keys.
 - `lustre_rpm_globs`: Optional list. Shell glob patterns for rpms to install. Note order is important as the built RPMs are not in a yum repo. Default is just the `kmod-lustre-client` and `lustre-client` packages.
-- `lustre_cleanup_build`: Optional bool. Whether to uninstall prerequisite packages and delete the build directories etc. Default `true`.
+- `lustre_build_cleanup`: Optional bool. Whether to uninstall prerequisite packages and delete the build directories etc. Default `true`.

ansible/roles/lustre/defaults/main.yml

@@ -1,8 +1,9 @@
 lustre_version: '2.15.5' # https://www.lustre.org/lustre-2-15-5-released/
+lustre_lnet_label: tcp
 #lustre_mgs_nid:
-lustre_fs_name: lustre
-lustre_mount_point: "/mnt/lustre/{{ lustre_fs_name }}"
+lustre_mounts: []
 lustre_mount_state: mounted
+lustre_mount_options: 'defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service'
 
 # below variables are for build and should not generally require changes
 lustre_build_packages:
@@ -32,4 +33,4 @@ lustre_configure_opts:
 lustre_rpm_globs: # NB: order is important here, as not installing from a repo
   - "kmod-lustre-client-{{ lustre_version | split('.') | first }}*" # only take part of the version as -RC versions produce _RC rpms
   - "lustre-client-{{ lustre_version | split('.') | first }}*"
-lustre_cleanup_build: true
+lustre_build_cleanup: true

ansible/roles/lustre/tasks/configure.yml

@@ -29,19 +29,23 @@
     name: lnet
     state: "{{ 'restarted' if _lnet_conf.changed else 'started' }}"
 
+# TODO: on some nodes we've got additional nodes!
+# An systemd lnet restart fixed it
+
 - name: Ensure mount points exist
   ansible.builtin.file:
-    path: "{{ lustre_mount_point }}"
+    path: "{{ item.mount_point }}"
     state: directory
-  when: "lustre_mount_state != 'absent'"
+  loop: "{{ lustre_mounts }}"
+  when: "(item.mount_state | default(lustre_mount_state)) != 'absent'"
 # TODO: do we need to set owner/group/mode either pre- or post-mount?
 
-# TODO: is this an ok way of doing it?
 - name: Mount lustre filesystem
   ansible.posix.mount:
     fstype: lustre
-    src: "{{ lustre_mgs_nid }}:/{{ lustre_fs_name }}"
-    path: "{{ lustre_mount_point }}"
-    state: "{{ lustre_mount_state }}"
-    opts: "defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service"
-    # opts are systemd defaults from http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes
+    src: "{{ lustre_mgs_nid }}:/{{ item.fs_name }}"
+    path: "{{ item.mount_point }}"
+    state: "{{ (item.mount_state | default(lustre_mount_state)) }}"
+    opts: "{{ item.mount_options | default(lustre_mount_options) }}"
+  loop: "{{ lustre_mounts }}"
+ 

ansible/roles/lustre/tasks/install.yml

@@ -67,4 +67,4 @@
     file:
       path: "{{ lustre_build_dir }}"
       state: absent
-  when: lustre_cleanup_build | bool
+  when: lustre_build_cleanup | bool

ansible/roles/lustre/tasks/validate.yml

@@ -13,15 +13,15 @@
     that: selinux_state in ['permissive', 'disabled']
     fail_msg: "SELinux must be permissive for Lustre not '{{ selinux_state }}'; see variable selinux_state"
 
-# TODO: fixme lustre_mounts
-# - name: Ensure lustre_mgs_nid is defined
-#   assert:
-#     that: item.mgs_nid | default(lustre_mgs_nid) is defined
-#     fail_msg: lustre_mounts entries must specify mgs_nid or lustre_mgs_nid must be defined
-#   loop: "{{ lustre_mounts }}"
+- name: Ensure lustre_mgs_nid is defined
+  assert:
+    that: lustre_mgs_nid is defined
+    fail_msg: Variable lustre_mgs_nid must be defined
 
-# - name: Ensure lustre_fs_name is defined
-#   assert:
-#     that: item.lustre_fs_name | default(lustre_fs_name) is defined
-#     fail_msg: lustre_mounts entries must specify fs_name or lustre_fs_name must be specified
-#   loop: "{{ lustre_mounts }}"
+- name: Ensure lustre_mounts entries define filesystem name and mount point
+  assert:
+    that:
+      - item.fs_name is defined
+      - item.mount_point is defined
+    fail_msg: All lustre_mounts entries must specify fs_name and mount_point
+  loop: "{{ lustre_mounts }}"

ansible/roles/lustre/templates/lnet.conf.j2

@@ -1,6 +1,6 @@
 net:
     - net type: tcp1
       local NI(s):
-        - nid: {{ _lustre_ip }}@tcp1
+        - nid: {{ _lustre_ip }}@{{ lustre_lnet_label }}
           interfaces:
               0: {{ _lustre_interface }}