auto gres v5 - proper top-level/overrride

Author: sjpb

defaults/main.yml

@@ -41,7 +41,7 @@ openhpc_default_config:
   PropagateResourceLimitsExcept: MEMLOCK
   Epilog: /etc/slurm/slurm.epilog.clean
   ReturnToService: 2
-  GresTypes: "{{ ohpc_nodegroup_gres_types if ohpc_nodegroup_gres_types != '' else 'omit' }}"
+  GresTypes: "{{ ohpc_gres_types | flatten | sort | unique | join(',') if ohpc_gres_types else 'omit' }}"
 openhpc_cgroup_default_config:
   ConstrainCores: "yes"
   ConstrainDevices: "yes"
@@ -50,19 +50,18 @@ openhpc_cgroup_default_config:
 
 openhpc_config: {}
 openhpc_cgroup_config: {}
-ohpc_nodegroup_gres_types: >-
-  {% if openhpc_gres_autodetect == 'nvml' %}
-  gpu
-  {% else %}
-  {{
-    openhpc_nodegroups |
-    community.general.json_query('[].gres[].conf') |
-    map('split', ':') |
-    map('first') |
-    unique |
-    join(',')
-  }}
-  {% endif %}
+ohpc_gres_types:
+  # toplevel nvml autodetect:
+  - "{{ ['gpu'] if openhpc_gres_autodetect == 'nvml' else [] }}"
+  # nodegroup nvml autodetect:
+  - "{{ ['gpu'] if openhpc_nodegroups | map(attribute='gres_autodetect', default='') | unique | select('eq', 'nvml') else [] }}"
+  # nodegroup specific gres conf:
+  - "{{
+        openhpc_nodegroups |
+        community.general.json_query('[].gres[].conf') |
+        map('regex_search', '^(\\w+)')
+    }}"
+
 openhpc_gres_template: gres.conf.j2
 openhpc_cgroup_template: cgroup.conf.j2
 

files/nodegroup.schema

@@ -21,15 +21,35 @@
   delegate_to: localhost
   run_once: true
 
-- name: Validate openhpc_nodegroups
-  ansible.utils.validate:
-    criteria: "{{ lookup('file', 'nodegroup.schema') }}"
-    engine: 'ansible.utils.jsonschema'
-    data: "{{ item }}"
-  vars:
-    ansible_jsonschema_draft: '2020-12'
+
+# - name: Validate openhpc_nodegroups contain name
+#   ansible.builtin.assert:
+#     that:
+#       - "'name' in nodegroup
+#       - openhpc_cluster_name ~ '_' ~ nodegroup.name in group_names
+#     fail_msg: "nodegroup does not specify a name or openhpc_cluster_name ~ '_' ~ nodegroup.name not found in inventory"
+
+#   loop: "{{ openhpc_nodegroups }}"
+
+- name: debug openhpc_nodegroups
+  ansible.builtin.debug:
+    msg: "{{ (item.gres | select('contains', 'file')) }} == {{ (item.gres | length) }}"
+  loop: "{{ openhpc_nodegroups }}"
+  when:
+    - item.gres_autodetect | default(openhpc_gres_autodetect) == 'off'
+    - "'gres' in item"
   delegate_to: localhost
+  run_once: true
+
+- name: Validate openhpc_nodegroups
+  ansible.builtin.assert:
+    that: "(item.gres | select('contains', 'file') | length) == (item.gres | length)"
+    fail_msg: "GRES configuration(s) in openhpc_nodegroup '{{ item.name }}' do not include 'file' but GRES autodetection is not enabled"
   loop: "{{ openhpc_nodegroups }}"
+  when:
+    - item.gres_autodetect | default(openhpc_gres_autodetect) == 'off'
+    - "'gres' in item"
+  delegate_to: localhost
   run_once: true
 
 - name: Fail if partition configuration is outdated

tasks/validate.yml

@@ -1,11 +1,10 @@
 AutoDetect={{ openhpc_gres_autodetect }}
 {% for nodegroup in openhpc_nodegroups %}
-{%      set nodegroup_gres_autodetect = nodegroup.gres_autodetect | default('off') %}
 {%      set inventory_group_name = openhpc_cluster_name ~ '_' ~ nodegroup.name %}
 {%      set inventory_group_hosts = groups.get(inventory_group_name, []) %}
 {%      set hostlist_string  = inventory_group_hosts | hostlist_expression | join(',') %}
 {%      for gres in nodegroup.gres | default([]) %}
 {%          set gres_name, gres_type, _ = gres.conf.split(':') %}
-NodeName={{ hostlist_string }} AutoDetect={{ nodegroup_gres_autodetect }} Name={{ gres_name }} Type={{ gres_type }}{% if openhpc_gres_autodetect == nodegroup_gres_autodetect == 'off' %} File={{ gres.file | mandatory('The gres configuration dictionary: ' ~  gres ~ ' is missing the file key, but gres_autodetect is not specified. The error occured on node group: ' ~  nodegroup.name ~ '. Please add the file key or set gres_autodetect.') }}{% endif %}
+NodeName={{ hostlist_string }}{% if 'gres_autodetect' in nodegroup %} AutoDetect={{ nodegroup.gres_autodetect }}{% endif %} Name={{ gres_name }} Type={{ gres_type }}{% if 'file' in gres %} File={{ gres.file }}{% endif %}
 {%      endfor %}{# gres #}
 {% endfor %}{# nodegroup #}