Reconfigure all clusters if slurn has topo issues

Author: arnaudfroidmont

bin/slurm_config.sh

@@ -19,6 +19,18 @@ then
       sudo /usr/sbin/slurmctld -c
    fi
    ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook $playbooks_path/slurm_config.yml
+   if [[ ${@: -1} == "--INITIAL" || ${@: -1} == "--initial" || ${@: -1} == "-INITIAL" || ${@: -1} == "-initial" ]]
+   then
+      for inventory in /opt/oci-hpc/autoscaling/clusters/*/inventory ; 
+      do
+         if [ -f $(dirname $inventory)/currently* ]
+         then
+            echo "Cluster is not in running state"
+         else
+            ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook $playbooks_path/slurm_config_as.yml -i $inventory
+         fi
+      done
+   fi
  else
    echo "There are some duplicates instance_keyword lines, please make them unique"   
 fi

playbooks/slurm_config_as.yml

@@ -0,0 +1,12 @@
+- hosts: compute
+  vars:
+    destroy: false
+    initial: false
+    download_path: "{{ '/nfs/cluster/' if cluster_nfs|bool else '/tmp' }}"
+    enroot_top_path: "{{ nvme_path }}/enroot/"
+  vars_files:
+    - "/opt/oci-hpc/conf/queues.conf"
+  tasks:
+    - include_role: 
+        name: slurm
+      when: slurm|default(false)|bool