Fixes gres on demand scheduling

By adding a full gres.conf to each node, the on demand scheduling is fixed. Also fixes bibilog and enables DebugFlag=Gres and TaskPlugin=task/cgroup

#726

Author: XaverStiensmeier

bibigrid/resources/bin/bibilog

@@ -8,7 +8,7 @@ fi
 if [ "$2" == "fail" ]; then
   fail_create="fail"
 else
-  fail_create="create"
+  fail_create="resume"
 fi
 
 LOG="/var/log/slurm/worker_logs/$fail_create/$err_out"

bibigrid/resources/defaults/slurm/slurm.j2

@@ -12,8 +12,7 @@ MpiDefault=none
 ProctrackType=proctrack/cgroup # linuxproc # changed for 23.11.0
 # ReturnToService=2
 SwitchType=switch/none
-TaskPlugin=task/none
-#TaskPlugin=task/cgroup
+TaskPlugin=task/cgroup
 JobAcctGatherType=jobacct_gather/linux
 
 # RESOURCES
@@ -65,6 +64,7 @@ SlurmctldDebug=debug # info
 SlurmctldLogFile=/var/log/slurm/slurmctld.log
 SlurmdDebug=info
 SlurmdLogFile=/var/log/slurm/slurmd.log
+DebugFlags=Gres
 
 # COMPUTE NODES
 # use_master_as_compute

bibigrid/resources/playbook/roles/bibigrid/handlers/main.yml

@@ -34,6 +34,14 @@
     state: restarted
   when: "'master' in group_names"
 
+- name: Enable Slurmd
+  systemd:
+    name: "slurmd"
+    enabled: true
+    masked: false
+    state: started
+    daemon_reload: true
+
 - name: Slurmd
   systemd:
     name: slurmd

bibigrid/resources/playbook/roles/bibigrid/tasks/042-slurm.yaml

@@ -108,12 +108,4 @@
     group: root
     mode: "0o444"
     force: true
-  when: flavor.gres is defined
-
-- name: Enable slurmd services
-  systemd:
-    name: "slurmd"
-    enabled: true
-    masked: false
-    state: started
-    daemon_reload: true
+  notify: Enable Slurmd

bibigrid/resources/playbook/roles/bibigrid/templates/slurm/gres.j2

@@ -1,8 +1,13 @@
 # GRES CONFIG
+{% for node_name in groups.master + groups.workers %}
+{% set node = hostvars[node_name] %}
 {% set ns = namespace(device_index=0) %}
-{% for gres in flavor.gres %}
+{% if node.flavor.gres is defined %}
+{% for gres in node.flavor.gres %}
 {% for i in range(gres.count | int) %}
-Name={{ gres.name }} Type={{ gres.type }} File=/dev/nvidia{{ ns.device_index }}
+NodeName={{ node.name }} Name={{ gres.name }} Type={{ gres.type }} File=/dev/nvidia{{ ns.device_index }}
 {% set ns.device_index = ns.device_index + 1 %}
 {% endfor %}
+{% endfor %}
+{% endif %}
 {% endfor %}

bibigrid/resources/playbook/roles/bibigrid/templates/slurm/gres_old_unused.j2

@@ -0,0 +1,8 @@
+# GRES CONFIG
+{% set ns = namespace(device_index=0) %}
+{% for gres in flavor.gres %}
+{% for i in range(gres.count | int) %}
+Name={{ gres.name }} Type={{ gres.type }} File=/dev/nvidia{{ ns.device_index }}
+{% set ns.device_index = ns.device_index + 1 %}
+{% endfor %}
+{% endfor %}