Merge branch 'main' into feat/additional-nodes

Author: sjpb

.github/workflows/extra.yml

@@ -32,11 +32,11 @@ jobs:
           - image_name: openhpc-extra-RL8
             source_image_name_key: RL8 # key into environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
             inventory_groups: doca,cuda,lustre
-            volume_size: 30 # needed for cuda
+            volume_size: 35 # needed for cuda
           - image_name: openhpc-extra-RL9
             source_image_name_key: RL9
             inventory_groups: doca,cuda,lustre
-            volume_size: 30 # needed for cuda
+            volume_size: 35 # needed for cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack

.gitignore

@@ -5,3 +5,4 @@ venv
 *.pyc
 packer/openhpc2
 .vscode
+requirements.yml.last

ansible/.gitignore

@@ -90,3 +90,7 @@ roles/*
 !roles/gateway/**
 !roles/alertmanager/
 !roles/alertmanager/**
+!roles/slurm_recompile/**
+!roles/slurm_recompile/**
+!roles/nhc/
+!roles/nhc/**

ansible/bootstrap.yml

@@ -134,6 +134,7 @@
 
 - hosts: dnf_repos
   become: yes
+  tags: dnf_repos
   tasks:
   - name: Check that creds won't be leaked to users
     ansible.builtin.assert:

ansible/disable-repos.yml

@@ -48,6 +48,20 @@
         name: cuda
         tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
 
+- name: Setup vGPU
+  hosts: vgpu
+  become: yes
+  gather_facts: yes
+  tags: vgpu
+  tasks:
+    - include_role:
+        name: stackhpc.linux.vgpu
+        tasks_from: "{{ 'configure.yml' if appliances_mode == 'configure' else 'install.yml' }}"
+  handlers:
+    - name: reboot
+      fail:
+        msg: Reboot handler for stackhpc.linux.vgpu role fired unexpectedly. This was supposed to be unreachable.
+
 - name: Persist hostkeys across rebuilds
   # Must be after filesystems.yml (for storage)
   # and before portal.yml (where OOD login node hostkeys are scanned)
@@ -58,17 +72,6 @@
     - import_role:
         name: persist_hostkeys
 
-
-- name: Setup NFS export for compute node configuration
-  hosts: compute_init:!builder
-  # NB: has to be after eeesi and os-manila-mount
-  tags: compute_init
-  become: yes
-  tasks:
-    - include_role:
-        name: compute_init
-        tasks_from: export.yml
-
 - name: Install k9s
   become: yes
   hosts: k9s

ansible/extras.yml

@@ -250,14 +250,24 @@
         name: cloudalchemy.grafana
         tasks_from: install.yml
 
+- name: Add support for NVIDIA GPU auto detection to Slurm
+  hosts: cuda
+  become: yes
+  tasks:
+    - name: Recompile slurm
+      import_role:
+        name: slurm_recompile
+      vars:
+        slurm_recompile_with_nvml: "{{ groups.cuda | length > 0 }}"
+
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
     hook_path: "{{ appliances_environment_root }}/hooks/post.yml"
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
-- import_playbook: disable-repos.yml
+- import_playbook: final.yml
 
 - hosts: builder
   become: yes

ansible/fatimage.yml

@@ -0,0 +1,18 @@
+- hosts: dnf_repos
+  become: yes
+  tags: dnf_repos
+  tasks:
+    - name: Disable pulp repos
+      ansible.builtin.include_role:
+        name: dnf_repos
+        tasks_from: disable_repos.yml
+
+- name: Setup NFS export for compute_init
+  hosts: compute_init:!builder
+  # NB: done last so other roles can prepare configuration etc
+  tags: compute_init
+  become: yes
+  tasks:
+    - include_role:
+        name: compute_init
+        tasks_from: export.yml

ansible/final.yml

@@ -75,6 +75,7 @@ it also requires an image build with the role name added to the
 | extras.yml               | basic_users             | All functionality [6]           | No                  |
 | extras.yml               | eessi                   | All functionality [7]           | No                  |
 | extras.yml               | cuda                    | None required - use image build | Yes [8]             |
+| extras.yml               | vgpu                    | All functionality               | Yes                 |
 | extras.yml               | persist_hostkeys        | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | compute_init (export)   | Not relevant for compute nodes  | n/a                 |
 | extras.yml               | k9s (install)           | Not relevant during boot        | n/a                 |
@@ -84,6 +85,7 @@ it also requires an image build with the role name added to the
 | slurm.yml                | openhpc [10]            | All slurmd functionality        | No                  |
 | slurm.yml                | (set memory limits)     | Fully supported                 | No                  |
 | slurm.yml                | (block ssh)             | Fully supported                 | No                  |
+| slurm.yml                | nhc                     | Fully supported                 | No                  |
 | portal.yml               | (openondemand server)   | Not relevant for compute nodes  | n/a                 |
 | portal.yml               | (openondemand vnc desktop)  | None required - use image build | No              |
 | portal.yml               | (openondemand jupyter server) | None required - use image build | No            |

ansible/roles/compute_init/README.md

@@ -19,6 +19,8 @@
     enable_basic_users: "{{ os_metadata.meta.basic_users | default(false) | bool }}"
     enable_eessi: "{{ os_metadata.meta.eessi | default(false) | bool }}"
     enable_chrony: "{{ os_metadata.meta.chrony | default(false) | bool }}"
+    enable_vgpu: "{{ os_metadata.meta.vpgu | default(false) | bool }}"
+    enable_nhc: "{{ os_metadata.meta.nhc | default(false) | bool }}"
 
     # TODO: "= role defaults" - could be moved to a vars_file: on play with similar precedence effects
     resolv_conf_nameservers: []
@@ -63,12 +65,12 @@
         mode: u=rX,g=rwX,o=
 
     - name: Mount /mnt/cluster
-      mount:
+      ansible.posix.mount:
         path: /mnt/cluster
         src: "{{ server_node_ip }}:/exports/cluster"
         fstype: nfs
         opts: ro,sync
-        state: mounted
+        state: ephemeral # will be unmounted after sync, don't want it in fstab
       register: _mount_mnt_cluster
       ignore_errors: true
       # exits from playbook if this failed below, allowing ansible-init to
@@ -295,6 +297,12 @@
             cmd: "cvmfs_config setup"
       when: enable_eessi
 
+    - name: Configure VGPUs
+      include_role:
+        name: stackhpc.linux.vgpu
+        tasks_from: 'configure.yml'
+      when: enable_vgpu
+
     # NB: don't need conditional block on enable_compute as have already exited
     # if not the case
     - name: Write Munge key
@@ -350,6 +358,11 @@
         enabled: true
         state: started
 
+    - name: Provide NHC configuration
+      ansible.builtin.include_role:
+        name: nhc
+        tasks_from: boot.yml
+      when: enable_nhc
 
     - name: Ensure node is resumed
       # TODO: consider if this is always safe for all job states?