Merge branch 'main' into fix-readme-tofu

Author: bertiethorpe

.github/workflows/stackhpc.yml

@@ -178,12 +178,13 @@ jobs:
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
 
-      - name: Test reimage of compute nodes and compute-init (via rebuild adhoc)
+      - name: Test compute node reboot and compute-init
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           ansible-playbook -v --limit compute ansible/adhoc/rebuild.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
+          ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
 
       - name: Check sacct state survived reimage
         run: |

ansible/.gitignore

@@ -80,3 +80,5 @@ roles/*
 !roles/slurm_stats/**
 !roles/pytools/
 !roles/pytools/**
+!roles/rebuild/
+!roles/rebuild/**

ansible/adhoc/reboot_via_slurm.yml

@@ -0,0 +1,24 @@
+# Reboot compute nodes via slurm. Nodes will be rebuilt if `image_id` in inventory is different to the currently-provisioned image.
+# Example:
+#   ansible-playbook -v ansible/adhoc/reboot_via_slurm.yml
+
+- hosts: login
+  run_once: true
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Submit a Slurm job to reboot compute nodes
+      ansible.builtin.shell: |
+        set -e
+        srun --reboot -N 2 uptime
+      become_user: root
+      register: slurm_result
+      failed_when: slurm_result.rc != 0
+
+    - name: Fetch Slurm controller logs if reboot fails
+      ansible.builtin.shell: |
+        journalctl -u slurmctld --since "10 minutes ago" | tail -n 50
+      become_user: root
+      register: slurm_logs
+      when: slurm_result.rc != 0
+      delegate_to: "{{ groups['control'] | first }}"

ansible/bootstrap.yml

@@ -126,7 +126,9 @@
     ansible.builtin.assert:
       that: dnf_repos_password is undefined
       fail_msg: Passwords should not be templated into repofiles during configure, unset 'dnf_repos_password'
-    when: appliances_mode == 'configure'
+    when:
+      - appliances_mode == 'configure'
+      - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
 - hosts: squid
   tags: squid

ansible/roles/compute_init/README.md

@@ -1,11 +1,109 @@
-# EXPERIMENTAL: compute-init
-
-Experimental / in-progress functionality to allow compute nodes to rejoin the
-cluster after a reboot.
-
-To enable this add compute nodes (or a subset of them into) the `compute_init`
-group.
-
+# EXPERIMENTAL: compute_init
+
+Experimental functionality to allow compute nodes to rejoin the cluster after
+a reboot without running the `ansible/site.yml` playbook.
+
+To enable this:
+1. Add the `compute` group (or a subset) into the `compute_init` group. This is
+   the default when using cookiecutter to create an environment, via the
+   "everything" template.
+2. Build an image which includes the `compute_init` group. This is the case
+   for StackHPC-built release images.
+3. Enable the required functionalities during boot, by setting the
+   `compute_init_enable` property for a compute group in the
+   OpenTofu `compute` variable to a list which includes "compute", plus the
+   other roles/functionalities required, e.g.:
+
+   ```terraform
+   ...
+   compute = {
+      general = {
+         nodes = ["general-0", "general-1"]
+         compute_init_enable = ["compute", ... ] # see below
+      }
+   }
+   ...
+   ```
+
+## Supported appliance functionalities
+
+In the table below, if a role is marked as supported then its functionality
+can be enabled during boot by adding the role name to the `compute_init_enable`
+property described above. If a role is marked as requiring a custom image then
+it also requires an image build with the role name added to the
+[Packer inventory_groups variable](../../../docs/image-build.md).
+
+| Playbook                 | Role (or functionality) | Support                         | Custom image reqd.? |
+| -------------------------|-------------------------|---------------------------------|---------------------|
+| hooks/pre.yml            | ?                       | None at present                 | n/a                 |
+| validate.yml             | n/a                     | Not relevant during boot        | n/a                 |
+| bootstrap.yml            | (wait for ansible-init) | Not relevant during boot        | n/a                 |
+| bootstrap.yml            | resolv_conf             | Fully supported                 | No                  |
+| bootstrap.yml            | etc_hosts               | Fully supported                 | No                  |
+| bootstrap.yml            | proxy                   | None at present                 | No                  |
+| bootstrap.yml            | (/etc permissions)      | None required - use image build | No                  |
+| bootstrap.yml            | (ssh /home fix)         | None required - use image build | No                  |
+| bootstrap.yml            | (system users)          | None required - use image build | No                  |
+| bootstrap.yml            | systemd                 | None required - use image build | No                  |
+| bootstrap.yml            | selinux                 | None required - use image build | Maybe [1]           |
+| bootstrap.yml            | sshd                    | None at present                 | No                  |
+| bootstrap.yml            | dnf_repos               | None at present [2]             | -                   |
+| bootstrap.yml            | squid                   | Not relevant for compute nodes  | n/a                 |
+| bootstrap.yml            | tuned                   | Fully supported                 | No                  |         
+| bootstrap.yml            | freeipa_server          | Not relevant for compute nodes  | n/a                 |
+| bootstrap.yml            | cockpit                 | None required - use image build | No                  |
+| bootstrap.yml            | firewalld               | Not relevant for compute nodes  | n/a                 |
+| bootstrap.yml            | fail2ban                | Not relevant for compute nodes  | n/a                 |
+| bootstrap.yml            | podman                  | Not relevant for compute nodes  | n/a                 |
+| bootstrap.yml            | update                  | Not relevant during boot        | n/a                 |
+| bootstrap.yml            | reboot                  | Not relevant for compute nodes  | n/a                 |
+| bootstrap.yml            | ofed                    | Not relevant during boot        | Yes                 |
+| bootstrap.yml            | ansible_init (install)  | Not relevant during boot        | n/a                 |
+| bootstrap.yml            | k3s (install)           | Not relevant during boot        | n/a                 |
+| hooks/post-bootstrap.yml | ?                       | None at present                 | n/a                 |
+| iam.yml                  | freeipa_client          | None at present [3]             | Yes                 |
+| iam.yml                  | freeipa_server          | Not relevant for compute nodes  | n/a                 |
+| iam.yml                  | sssd                    | None at present                 | No                  |
+| filesystems.yml          | block_devices           | None required - role deprecated | n/a                 |
+| filesystems.yml          | nfs                     | All client functionality        | No                  |
+| filesystems.yml          | manila                  | All functionality               | No [4]              |
+| filesystems.yml          | lustre                  | None at present                 | Yes                 |
+| extras.yml               | basic_users             | All functionality [5]           | No                  |
+| extras.yml               | eessi                   | All functionality [6]           | No                  |
+| extras.yml               | cuda                    | None required - use image build | Yes [7]             |
+| extras.yml               | persist_hostkeys        | Not relevant for compute nodes  | n/a                 |
+| extras.yml               | compute_init (export)   | Not relevant for compute nodes  | n/a                 |
+| extras.yml               | k9s (install)           | Not relevant during boot        | n/a                 |
+| extras.yml               | extra_packages          | None at present [8]             | -                   |
+| slurm.yml                | mysql                   | Not relevant for compute nodes  | n/a                 |
+| slurm.yml                | rebuild                 | Not relevant for compute nodes  | n/a                 |
+| slurm.yml                | openhpc [9]             | All slurmd functionality        | No                  |
+| slurm.yml                | (set memory limits)     | None at present                 | -                   |
+| slurm.yml                | (block ssh)             | None at present                 | -                   |
+| portal.yml               | (openondemand server)   | Not relevant for compute nodes  | n/a                 |
+| portal.yml               | (openondemand vnc desktop)  | None required - use image build | No              |
+| portal.yml               | (openondemand jupyter server) | None required - use image build | No            |
+| monitoring.yml           | node_exporter           | None required - use image build | No                  |
+| monitoring.yml           | (other  monitoring)     | Not relevant for compute nodes  | -                   |
+| disable-repos.yml        | dnf_repos               | None at present [2]             | -                   |
+| hooks/post.yml           | ?                       | None at present                 | -                   |
+
+
+Notes:
+1. `selinux` is set to disabled in StackHPC images.
+2. Requirement for this functionality is TBD.
+3. FreeIPA client functionality would be better provided using a client fork
+   which uses pkinit keys rather than OTP to reenrol nodes.
+4. Assuming default Ceph client version.
+5. Assumes home directory already exists on shared storage.
+6. Assumes `cvmfs_config` is the same on control node and all compute nodes.
+7. If `cuda` role was run during build, the nvidia-persistenced is enabled
+  and will start during boot.
+8. Would require `dnf_repos`.
+9. `openhpc` does not need to be added to `compute_init_enable`, this is
+   automatically enabled by adding `compute`.
+
+## Approach
 This works as follows:
 1. During image build, an ansible-init playbook and supporting files
 (e.g. templates, filters, etc) are installed.
@@ -31,21 +129,7 @@ The check in 4b. above is what prevents the compute-init script from trying
 to configure the node before the services on the control node are available
 (which requires running the site.yml playbook).
 
-The following roles/groups are currently fully functional:
-- `resolv_conf`: all functionality
-- `etc_hosts`: all functionality
-- `nfs`: client functionality only
-- `manila`: all functionality
-- `basic_users`: all functionality, assumes home directory already exists on
-  shared storage
-- `eessi`: all functionality, assumes `cvmfs_config` is the same on control
-  node and all compute nodes.
-- `openhpc`: all functionality
-
-The above may be enabled by setting the compute_init_enable property on the
-tofu compute variable.
-
-# Development/debugging
+## Development/debugging
 
 To develop/debug changes to the compute script without actually having to build
 a new image:
@@ -83,7 +167,7 @@ reimage the compute node(s) first as in step 2 and/or add additional metadata
 as in step 3.
 
 
-# Design notes
+## Design notes
 - Duplicating code in roles into the `compute-init` script is unfortunate, but
   does allow developing this functionality without wider changes to the
   appliance.

ansible/roles/compute_init/files/compute-init.yml

@@ -9,6 +9,7 @@
     enable_compute: "{{ os_metadata.meta.compute | default(false) | bool }}"
     enable_resolv_conf: "{{ os_metadata.meta.resolv_conf | default(false) | bool }}"
     enable_etc_hosts: "{{ os_metadata.meta.etc_hosts | default(false) | bool }}"
+    enable_tuned:  "{{ os_metadata.meta.tuned | default(false) | bool }}"
     enable_nfs: "{{ os_metadata.meta.nfs | default(false) | bool }}"
     enable_manila: "{{ os_metadata.meta.manila | default(false) | bool }}"
     enable_basic_users: "{{ os_metadata.meta.basic_users | default(false) | bool }}"
@@ -17,6 +18,12 @@
     # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
     resolv_conf_nameservers: []
 
+    tuned_profile_baremetal: hpc-compute
+    tuned_profile_vm: virtual-guest
+    tuned_profile: "{{ tuned_profile_baremetal if ansible_virtualization_role != 'guest' else tuned_profile_vm }}"
+    tuned_enabled: true
+    tuned_started: true
+
     nfs_client_mnt_point: "/mnt"
     nfs_client_mnt_options:
     nfs_client_mnt_state: mounted
@@ -59,9 +66,9 @@
       file:
         path: /mnt/cluster
         state: directory
-        owner: root
+        owner: slurm
         group: root
-        mode: u=rwX,go= # is sensitive
+        mode: u=rX,g=rwX,o=
 
     - name: Mount /mnt/cluster
       mount:
@@ -125,6 +132,10 @@
         mode: 0644
       when: enable_etc_hosts
 
+    - name: Configure tuned
+      include_tasks: tasks/tuned.yml
+      when: enable_tuned
+
     # NFS client mount
     - name: If nfs-clients is present
       include_tasks: tasks/nfs-clients.yml
@@ -276,6 +287,27 @@
         enabled: true
         state: started
 
+    - name: Set locked memory limits on user-facing nodes
+      lineinfile:
+        path: /etc/security/limits.conf
+        regexp: '\* soft memlock unlimited'
+        line: "* soft memlock unlimited"
+
+    - name: Configure sshd pam module
+      blockinfile:
+        path: /etc/pam.d/sshd
+        insertafter: 'account\s+required\s+pam_nologin.so'
+        block: |
+          account    sufficient   pam_access.so
+          account    required     pam_slurm.so
+
+    - name: Configure login access control
+      blockinfile:
+        path: /etc/security/access.conf
+        block: |
+          +:adm:ALL
+          -:ALL:ALL
+
     - name: Ensure node is resumed
       # TODO: consider if this is always safe for all job states?
       command: scontrol update state=resume nodename={{ ansible_hostname }}

ansible/roles/compute_init/tasks/export.yml

@@ -2,9 +2,9 @@
   file:
     path: /exports/cluster
     state: directory
-    owner: root
+    owner: slurm
     group: root
-    mode: u=rwX,go=
+    mode: u=rX,g=rwX,o=
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
 
@@ -23,21 +23,27 @@
   file:
     path: /exports/cluster/hostvars/{{ inventory_hostname }}/
     state: directory
-    mode: u=rwX,go=
-    # TODO: owner,mode,etc
+    owner: slurm
+    group: root
+    mode: u=rX,g=rwX,o=
   delegate_to: "{{ groups['control'] | first }}"
 
 - name: Template out hostvars
   template:
     src: hostvars.yml.j2
     dest: /exports/cluster/hostvars/{{ inventory_hostname }}/hostvars.yml
-    mode: u=rw,go=
+    owner: slurm
+    group: root
+    mode: u=r,g=rw,o=
   delegate_to: "{{ groups['control'] | first }}"
 
 - name: Copy manila share info to /exports/cluster
   copy:
     content: "{{ os_manila_mount_share_info_var | to_nice_yaml }}"
     dest: /exports/cluster/manila_share_info.yml
+    owner: root
+    group: root
+    mode: u=rw,g=r
   run_once: true
   delegate_to: "{{ groups['control'] | first }}"
   when: os_manila_mount_share_info is defined

ansible/roles/compute_init/tasks/install.yml

@@ -32,6 +32,8 @@
       dest: files/NetworkManager-dns-none.conf
     - src: ../../basic_users/filter_plugins/filter_keys.py
       dest: filter_plugins/filter_keys.py
+    - src: ../../tuned/tasks/configure.yml
+      dest: tasks/tuned.yml
     - src: ../../stackhpc.nfs/tasks/nfs-clients.yml
       dest: tasks/nfs-clients.yml
 

ansible/roles/lustre/tasks/configure.yml

@@ -1,19 +1,18 @@
 - name: Gather Lustre interface info
   shell:
     cmd: |
-      ip r get {{ _lustre_mgs_ip }}
+      ip --json r get {{ _lustre_mgs_ip }}
   changed_when: false
   register: _lustre_ip_r_mgs
   vars:
     _lustre_mgs_ip: "{{ lustre_mgs_nid | split('@') | first }}"
 
 - name: Set facts for Lustre interface
   set_fact:
-    _lustre_interface: "{{ _lustre_ip_r_mgs_info[4] }}"
-    _lustre_ip: "{{ _lustre_ip_r_mgs_info[6] }}"
+    _lustre_interface: "{{ _lustre_ip_r_mgs_info.dev }}"
+    _lustre_ip: "{{ _lustre_ip_r_mgs_info.prefsrc }}"
   vars:
-    _lustre_ip_r_mgs_info: "{{ _lustre_ip_r_mgs.stdout_lines.0 | split }}"
-    # first line e.g. "10.167.128.1 via 10.179.0.2 dev eth0 src 10.179.3.149 uid 1000"
+    _lustre_ip_r_mgs_info: "{{ _lustre_ip_r_mgs.stdout | from_json | first }}"
 
 - name: Write LNet configuration file
   template:
@@ -44,4 +43,3 @@
     state: "{{ (item.mount_state | default(lustre_mount_state)) }}"
     opts: "{{ item.mount_options | default(lustre_mount_options) }}"
   loop: "{{ lustre_mounts }}"
- 

ansible/roles/lustre/tasks/validate.yml

@@ -1,8 +1,3 @@
-- name: Assert using RockyLinux 9
-  assert:
-    that: ansible_distribution_major_version | int == 9
-    fail_msg: The 'lustre' role requires RockyLinux 9
-
 - name: Check kernel-devel package is installed
   command: "dnf list --installed kernel-devel-{{ ansible_kernel }}"
   changed_when: false