Support compute node rebuild/reboot via Slurm RebootProgram

ansible/.gitignore

@@ -80,3 +80,5 @@ roles/*
 !roles/slurm_stats/**
 !roles/pytools/
 !roles/pytools/**
+!roles/rebuild/
+!roles/rebuild/**

ansible/roles/rebuild/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+openhpc_rebuild_clouds: ~/.config/openstack/clouds.yaml

ansible/roles/rebuild/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Create /etc/openstack
+  file:
+    path: /etc/openstack
+    state: directory
+    owner: root
+    group: root
+    mode: '0400'
+
+- name: Copy out clouds.yaml
+  copy:
+    src: "{{ openhpc_rebuild_clouds }}"
+    dest: /etc/openstack/clouds.yaml
+    owner: root
+    group: root
+    mode: '0400'
+
+- name: Setup slurm tools
+  include_role:
+    name: slurm_tools

ansible/slurm.yml

@@ -9,6 +9,15 @@
     - include_role:
         name:  mysql
 
+- name: Setup slurm-driven rebuild
+  hosts: rebuild
+  become: yes
+  tags:
+    - rebuild
+  tasks:
+    - import_role:
+        name: rebuild
+
 - name: Setup slurm
   hosts: openhpc
   become: yes

environments/.stackhpc/inventory/extra_groups

@@ -1,10 +1,6 @@
 [basic_users:children]
 cluster
 
-[rebuild:children]
-control
-compute
-
 [etc_hosts:children]
 cluster
 
@@ -35,3 +31,6 @@ builder
 [sssd:children]
 # Install sssd into fat image
 builder
+
+[rebuild:children]
+control

environments/common/inventory/group_vars/all/openhpc.yml

@@ -35,8 +35,12 @@ openhpc_config_default:
   SlurmctldParameters:
     - enable_configless
   TaskPlugin: task/cgroup,task/affinity
+openhpc_config_reboot:
+  RebootProgram: /opt/slurm-tools/bin/slurm-openstack-rebuild
+  SlurmctldParameters:
+    - reboot_from_controller
 openhpc_config_extra: {}
-openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_extra, list_merge='append') }}"
+openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_reboot if groups['rebuild'] | length > 0 else {}, openhpc_config_extra, list_merge='append') }}"
 openhpc_state_save_location: "{{ appliances_state_dir + '/slurmctld' if appliances_state_dir is defined else '/var/spool' }}"
 
 openhpc_install_type: ohpc # 'ohpc' or 'generic', see https://github.com/stackhpc/ansible-slurm-appliance/pull/326

environments/common/inventory/groups

@@ -62,9 +62,6 @@ mysql
 # All hosts requiring control of SELinux status.
 cluster
 
-[rebuild]
-# Enable rebuild of nodes on an OpenStack cloud; add 'control' group plus 'compute' group or a subset of it.
-
 [update]
 # All hosts to (optionally) run yum update on.
 
@@ -165,3 +162,6 @@ extra_packages
 
 [pulp]
 # Add builder to this group to enable automatically syncing of pulp during image build
+
+[rebuild]
+# Enable rebuild of nodes on an OpenStack cloud; add 'control' group.

environments/common/layouts/everything

@@ -25,8 +25,6 @@ control
 [filebeat:children]
 slurm_stats
 
-# NB: [rebuild] not defined here as this template is used in CI
-
 [update:children]
 
 [fail2ban:children]
@@ -111,3 +109,5 @@ control
 [extra_packages:children]
 # Hosts to install specified additional packages on
 builder
+
+# NB: [rebuild] not defined here as this template is used in CI