Check if a node needs to be drained/resumed before actually doing it

Author: Bharat Kunwar

defaults/main.yml

@@ -5,6 +5,8 @@ openhpc_slurm_control_host:
 openhpc_slurm_partitions: []
 openhpc_cluster_name:
 openhpc_packages: []
+openhpc_drain_timeout: 86400
+openhpc_resume_timeout: 300
 openhpc_enable:
   control: false
   batch: false

tasks/drain.yml

@@ -6,14 +6,19 @@
 # - node_to_drain: compute node to drain
 # - drain_timeout: seconds to wait for node to drain, default is 86400.
 
+- name: Get nodes in DRAINED state
+  command: "sinfo --noheader --Node --format='%N' --states=DRAINED"
+  register: drained_nodes_results
+  changed_when: false
+
 - name: Drain compute node
-  become: yes
   command: "scontrol update nodename={{ inventory_hostname }} state=DRAIN reason='maintenance'"
+  when: inventory_hostname not in drained_nodes_results.stdout_lines
 
 - name: Check node has drained
   command: "sinfo --noheader --Node --format='%N' --states=DRAINED"
   register: drained_nodes
   until: "inventory_hostname in drained_nodes.stdout_lines"
   delay: 10
-  retries: "{{ (drain_timeout|default(86400) / 10)|int }}"
+  retries: "{{ (openhpc_drain_timeout/10) | int }}"
   changed_when: false

tasks/main.yml

@@ -10,7 +10,9 @@
 
 - include: drain.yml
   when: openhpc_enable.drain | default(false) | bool
+  delegate_to: "{{ openhpc_slurm_control_host }}"
 
 - include: resume.yml
   when: openhpc_enable.resume | default(false) | bool
+  delegate_to: "{{ openhpc_slurm_control_host }}"
 ...

tasks/resume.yml

@@ -6,14 +6,19 @@
 # - nodes_to_resume: compute node to resume
 # - resume_timeout: seconds to wait for node to resume, default is 300.
 
+- name: Get nodes in ALLOC,IDLE states
+  command: "sinfo --noheader --Node --format='%N' --states=ALLOC,IDLE"
+  register: resumed_nodes_results
+  changed_when: false
+
 - name: Resume compute node
-  become: yes
   command: "scontrol update nodename={{ inventory_hostname }} state=RESUME"
+  when: inventory_hostname not in resumed_nodes_results.stdout_lines
 
 - name: Check node has resumed
   command: "sinfo --noheader --Node --format='%N' --states=ALLOC,IDLE"
   register: resumed_nodes
   until: "inventory_hostname in resumed_nodes.stdout_lines"
   delay: 10
-  retries: "{{ (resume_timeout|default(300) / 10)|int }}"
+  retries: "{{ (openhpc_resume_timeout/10) | int }}"
   changed_when: false