Fix small issue with destroy race condition

Author: arnaudfroidmont

playbooks/roles/slurm/tasks/destroy-rack-aware.yml

@@ -57,6 +57,24 @@
   set_fact: nodes_to_remove="{{nodes_to_remove_temp_results.results | map(attribute='ansible_facts.nodes_to_remove_temp') | list}}"
   run_once: true
 
+- name: Get new inactive_nodes list
+  command: "scontrol show hostlistsorted {{inactive_list | union(nodes_to_remove) | join(',')}}"
+  register: new_inactive_list
+  run_once: true
+  delegate_to: 127.0.0.1
+
+- name: Adding nodes to inactive
+  vars:
+    - keyword: "{% for partition in queues %}{% for instance in partition.instance_types %}{% if instance.name == instance_type %}{{instance.instance_keyword}}{% endif %}{% endfor %}{% endfor %}"
+  become: true
+  lineinfile:
+    path: "{{ slurm_conf_path }}/topology.conf"
+    regexp: "SwitchName=inactive-{{queue}}-{{keyword}}\\sNodes.*"
+    line: "SwitchName=inactive-{{queue}}-{{keyword}} Nodes={{new_inactive_list.stdout }}"
+    state: present
+  run_once: true
+  delegate_to: 127.0.0.1
+  
 - name: Run the script to get the RackID
   shell: 'curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v1/host | jq .rackId'
 #  shell: echo $RANDOM | md5sum | head -c 20
@@ -160,26 +178,9 @@
   delegate_to: 127.0.0.1
   when: racks_left_list | list | length > 0
 
-- name: Get new inactive_nodes list
-  command: "scontrol show hostlistsorted {{inactive_list | union(nodes_to_remove) | join(',')}}"
-  register: new_inactive_list
-  run_once: true
-  delegate_to: 127.0.0.1
-
-- name: Adding nodes to inactive
-  vars:
-    - keyword: "{% for partition in queues %}{% for instance in partition.instance_types %}{% if instance.name == instance_type %}{{instance.instance_keyword}}{% endif %}{% endfor %}{% endfor %}"
-  become: true
-  lineinfile:
-    path: "{{ slurm_conf_path }}/topology.conf"
-    regexp: "SwitchName=inactive-{{queue}}-{{keyword}}\\sNodes.*"
-    line: "SwitchName=inactive-{{queue}}-{{keyword}} Nodes={{new_inactive_list.stdout }}"
-    state: present
-  run_once: true
-  delegate_to: 127.0.0.1
-
 - name: Reconfigure Slurm for topology
   become: true
   command: "scontrol reconfigure"
   delegate_to: 127.0.0.1
-  run_once: true
+  run_once: true
+  ignore_errors: yes

playbooks/roles/slurm/tasks/destroy.yml

@@ -120,4 +120,5 @@
   become: true
   command: "scontrol reconfigure"
   delegate_to: 127.0.0.1
-  run_once: true
+  run_once: true
+  ignore_errors: true