slurm controlled rebuild docs & adhoc support

Author: sjpb

ansible/adhoc/reboot_via_slurm.yml

@@ -0,0 +1,17 @@
+# Rebuild compute nodes via slurm.
+# Nodes will be rebuilt if `image_id` in inventory is different to the
+# currently-provisioned image. Otherwise they are rebooted.
+
+# Example:
+#   ansible-playbook -v ansible/adhoc/rebuild-via-slurm.yml
+
+# See docs/slurm-controlled-rebuild.md.
+
+- hosts: login
+  run_once: true
+  gather_facts: no
+  tasks:
+    - name: Run slurm-controlled rebuild
+      import_role:
+        name: rebuild
+        tasks_from: rebuild.yml

ansible/adhoc/rebuild-via-slurm.yml

@@ -1,12 +1,11 @@
 # EXPERIMENTAL: compute_init
 
-Experimental functionality to allow compute nodes to rejoin the cluster after
-a reboot without running the `ansible/site.yml` playbook.
+Allow compute nodes to rejoin the cluster after a reboot without running the
+`ansible/site.yml` playbook.
 
-**CAUTION:** The approach used here of exporting cluster secrets over NFS
-is considered to be a security risk due to the potential for cluster users to
-mount the share on a user-controlled machine by tunnelling through a login
-node. This feature should not be enabled on production clusters at this time.
+> [!NOTE]
+> This functionality is marked as experimental as it may be incomplete and the
+> required configuration may change with further development.
 
 To enable this:
 1. Add the `compute` group (or a subset) into the `compute_init` group.

ansible/roles/compute_init/README.md

@@ -1,30 +1,55 @@
 rebuild
 =========
 
-Enables reboot tool from https://github.com/stackhpc/slurm-openstack-tools.git to be run from control node.
+Enables reboot tool from https://github.com/stackhpc/slurm-openstack-tools.git
+to be run from control node.
 
 Requirements
 ------------
 
-clouds.yaml file
+An OpenStack clouds.yaml file containing credentials for a cloud under the
+"openstack" key.
 
 Role Variables
 --------------
 
-- `openhpc_rebuild_clouds`: Directory. Path to clouds.yaml file.
+The below is only used by this role's `main.yml` task file, i.e. when running
+the `ansible/site.yml` or `ansible/slurm.yml` playbooks:
 
+- `rebuild_clouds_path`: Optional. Path to `clouds.yaml` file on the deploy
+  host, default `~/.config/openstack/clouds.yaml`.
 
-Example Playbook
-----------------
+The below are only used by this role's `rebuild.yml` task file, i.e. when
+running the `ansible/adhoc/rebuild-via-slurm.yml` playbook:
 
-    - hosts: control
-      become: yes
-      tasks:
-        - import_role:
-            name: rebuild
+- `rebuild_job_partitions`: Optional. Comma-separated list of names of rebuild
+  partitions defined in `openhpc_slurm_partitions`. Useful as an extra-var for
+  limiting rebuilds. Default `rebuild`.
 
-License
--------
+- `rebuild_job_name`: Optional. Name of rebuild jobs. Default is `rebuild-`
+  suffixed with the node name.
 
-Apache-2.0
+- `rebuild_job_command`: Optional. String giving command to run in job after
+  node has been rebuilt. Default is to sleep for 5 seconds. Note job output is
+  send to `/dev/null` by default, as the root user running this has no shared
+  directory for job output.
 
+- `rebuild_job_reboot`: Bool, whether to add the `--reboot` flag to the job
+  to actually trigger a rebuild. Useful for e.g. testing priorities. Default
+  `true`.
+
+- `rebuild_job_options`: Optional. A string giving any other options to pass to
+  [sbatch](https://slurm.schedmd.com/sbatch.html). Default is empty string.
+
+- `rebuild_job_user`: Optional. The user to run the rebuild setup and job as.
+  Default `root`.
+
+- `rebuild_job_template`: Optional. The string to use to submit the job. See
+  [defaults.yml](defaults/main.yml).
+
+- `rebuild_job_hostlist`: String with a Slurm hostlist expression to restrict
+  a rebuild to only those nodes (e.g. `tux[1-3]` or `tux1,tux2`). If set,
+  `rebuild_partitions` must only define a single partition and that partition
+  must contain those nodes. Not for routine use, but may be useful to e.g.
+  reattempt a rebuild if this failed on specific nodes. Default is all nodes
+  in the relevant partition.

ansible/roles/rebuild/README.md

@@ -1,2 +1,23 @@
 ---
-openhpc_rebuild_clouds: ~/.config/openstack/clouds.yaml
+
+rebuild_clouds_path: ~/.config/openstack/clouds.yaml
+
+rebuild_job_partitions: rebuild
+rebuild_job_name: "rebuild-{{ item }}" # item is nodename
+rebuild_job_command: 'sleep 5'
+rebuild_job_reboot: true
+rebuild_job_options: ''
+rebuild_job_user: root
+rebuild_job_template: >-
+  sbatch
+  --nodelist={{ item }}
+  {{ '--reboot' if rebuild_job_reboot | bool else '' }}
+  --job-name={{ rebuild_job_name }}
+  --nodes=1
+  --exclusive
+  --partition={{ _rebuild_job_current_partition }}
+  --no-requeue
+  --output=/dev/null
+  --wrap="{{ rebuild_job_command }}"
+  {{ rebuild_job_options }}
+#rebuild_job_hostlist:

ansible/roles/rebuild/defaults/main.yml

@@ -10,7 +10,7 @@
 
 - name: Copy out clouds.yaml
   copy:
-    src: "{{ openhpc_rebuild_clouds }}"
+    src: "{{ rebuild_clouds_path }}"
     dest: /etc/openstack/clouds.yaml
     owner: slurm
     group: root

ansible/roles/rebuild/tasks/main.yml

@@ -0,0 +1,11 @@
+- name: Create rebuild jobs for partition
+  include_tasks:
+    file: rebuild_partition.yml
+  args:
+    apply:
+      become: yes
+      become_user: "{{ rebuild_job_user }}"
+  loop: "{{ rebuild_job_partitions | split(',') }}"
+  loop_control:
+    loop_var: _rebuild_job_current_partition
+

ansible/roles/rebuild/tasks/rebuild.yml

@@ -0,0 +1,21 @@
+- name: Get list of nodes in partition
+  ansible.builtin.command:
+    cmd: >-
+      sinfo
+      --Node
+      --Format=NodeList
+      --noheader
+      --partition={{ _rebuild_job_current_partition }}
+  register: _sinfo_partition
+  when: rebuild_job_hostlist is not defined
+
+- name: Expand rebuild_job_hostlist to host names
+  ansible.builtin.command:
+    cmd: "scontrol show hostnames {{ rebuild_job_hostlist }}"
+  register: _scontrol_hostnames
+  when: rebuild_job_hostlist is defined
+
+- name: Submit rebuild jobs
+  ansible.builtin.command:
+    cmd: "{{ rebuild_job_template }}"
+  loop: "{{ _scontrol_hostnames.stdout_lines | default(_sinfo_partition.stdout_lines) }}"