Merge remote-tracking branch 'origin/feat/nodegroups' into HEAD

jovial · jovial · commit 4238c704b1e8 · 2025-05-08T14:09:24.000+01:00
diff --git a/README.md b/README.md
@@ -52,30 +52,44 @@ each list element:
 
 ### slurm.conf
 
-`openhpc_slurm_partitions`: Optional. List of one or more slurm partitions, default `[]`.  Each partition may contain the following values:
-* `groups`: If there are multiple node groups that make up the partition, a list of group objects can be defined here.
-  Otherwise, `groups` can be omitted and the following attributes can be defined in the partition object:
-  * `name`: The name of the nodes within this group.
-  * `cluster_name`: Optional.  An override for the top-level definition `openhpc_cluster_name`.
-  * `extra_nodes`: Optional. A list of additional node definitions, e.g. for nodes in this group/partition not controlled by this role. Each item should be a dict, with keys/values as per the ["NODE CONFIGURATION"](https://slurm.schedmd.com/slurm.conf.html#lbAE) docs for slurm.conf. Note the key `NodeName` must be first.
-  * `ram_mb`: Optional.  The physical RAM available in each node of this group ([slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `RealMemory`) in MiB. This is set using ansible facts if not defined, equivalent to `free --mebi` total * `openhpc_ram_multiplier`.
-  * `ram_multiplier`: Optional.  An override for the top-level definition `openhpc_ram_multiplier`. Has no effect if `ram_mb` is set.
+`openhpc_nodegroups`: Optional, default `[]`. List of mappings, each defining a
+unique set of homogenous nodes:
+  * `name`: Required. Name of node group.
+  * `ram_mb`: Optional.  The physical RAM available in each node of this group
+  ([slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `RealMemory`)
+  in MiB. This is set using ansible facts if not defined, equivalent to
+  `free --mebi` total * `openhpc_ram_multiplier`.
+  * `ram_multiplier`: Optional.  An override for the top-level definition
+  `openhpc_ram_multiplier`. Has no effect if `ram_mb` is set.
   * `gres`: Optional. List of dicts defining [generic resources](https://slurm.schedmd.com/gres.html). Each dict should define:
       - `conf`: A string with the [resource specification](https://slurm.schedmd.com/slurm.conf.html#OPT_Gres_1) but requiring the format `<name>:<type>:<number>`, e.g. `gpu:A100:2`. Note the `type` is an arbitrary string.
-      - `file`: Omit if `openhpc_gres_autodetect` is set. A string with the [File](https://slurm.schedmd.com/gres.conf.html#OPT_File) (path to device(s)) for this resource, e.g. `/dev/nvidia[0-1]` for the above example.
-
+      - `file`: Omit if `gres_autodetect` is set. A string with the [File](https://slurm.schedmd.com/gres.conf.html#OPT_File) (path to device(s)) for this resource, e.g. `/dev/nvidia[0-1]` for the above example.
     Note [GresTypes](https://slurm.schedmd.com/slurm.conf.html#OPT_GresTypes) must be set in `openhpc_config` if this is used.
-
-* `default`: Optional.  A boolean flag for whether this partion is the default.  Valid settings are `YES` and `NO`.
-* `maxtime`: Optional.  A partition-specific time limit following the format of [slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `MaxTime`.  The default value is
+  * `node_params`: Optional. Mapping of additional parameters and values for
+  [node configuration](https://slurm.schedmd.com/slurm.conf.html#lbAE).
+
+  Each nodegroup will contain hosts from an Ansible inventory group named
+  `{{ openhpc_cluster_name }}_{{ group_name}}`. Note that:
+  - Each host may only appear in one nodegroup.
+  - Hosts in a nodegroup are assumed to be homogenous in terms of processor and memory.
+  - Hosts may have arbitrary hostnames, but these should be lowercase to avoid a
+    mismatch between inventory and actual hostname.
+  - An inventory group may be missing or empty, in which case the node group
+    contains no hosts.
+  - If the inventory group is not empty the play must contain at least one host.
+    This is used to set `Sockets`, `CoresPerSocket`, `ThreadsPerCore` and
+    optionally `RealMemory` for the nodegroup.
+
+`openhpc_partitions`: Optional, default `[]`. List of mappings, each defining a
+partition. Each partition mapping may contain:
+  * `name`: Required. Name of partition.
+  * `groups`: Optional. List of nodegroup names. If omitted, the partition name
+  is assumed to match a nodegroup name.
+  * `default`: Optional.  A boolean flag for whether this partion is the default.  Valid settings are `YES` and `NO`.
+  * `maxtime`: Optional.  A partition-specific time limit following the format of [slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `MaxTime`.  The default value is
   given by `openhpc_job_maxtime`. The value should be quoted to avoid Ansible conversions.
-* `partition_params`: Optional. Mapping of additional parameters and values for [partition configuration](https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION).
-
-For each group (if used) or partition any nodes in an ansible inventory group `<cluster_name>_<group_name>` will be added to the group/partition. Note that:
-- Nodes may have arbitrary hostnames but these should be lowercase to avoid a mismatch between inventory and actual hostname.
-- Nodes in a group are assumed to be homogenous in terms of processor and memory.
-- An inventory group may be empty or missing, but if it is not then the play must contain at least one node from it (used to set processor information).
-
+  * `partition_params`: Optional. Mapping of additional parameters and values for
+  [partition configuration](https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION).
 
 `openhpc_job_maxtime`: Maximum job time limit, default `'60-0'` (60 days). See [slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `MaxTime` for format. The default is 60 days. The value should be quoted to avoid Ansible conversions.
 
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -4,7 +4,8 @@ openhpc_slurm_service_started: "{{ openhpc_slurm_service_enabled }}"
 openhpc_slurm_service:
 openhpc_slurm_control_host: "{{ inventory_hostname }}"
 #openhpc_slurm_control_host_address:
-openhpc_slurm_partitions: []
+openhpc_partitions: []
+openhpc_nodegroups: []
 openhpc_cluster_name:
 openhpc_packages:
   - slurm-libpmi-ohpc
diff --git a/templates/gres.conf.j2 b/templates/gres.conf.j2
@@ -1,29 +1,17 @@
 AutoDetect=off
-{% set donehosts = [] | unique %}
-{% for part in openhpc_slurm_partitions %}
-{%  set nodelist = [] %}
-{%  for group in part.get('groups', [part]) %}
-{%      if 'gres' in group %}
-{%          set group_name = group.cluster_name|default(openhpc_cluster_name) ~ '_' ~ group.name %}
-{%          set inventory_group_hosts = groups.get(group_name, []) %}
-{%          set autodetect_mechanisms = inventory_group_hosts | group_by_gres_autodetect %}
-{%          for mechanism, _mechanism_hosts in autodetect_mechanisms.items() %}
-{%              set mechanism_hosts = _mechanism_hosts | difference(donehosts) %}
-{%              if mechanism != 'off' %}
-{%                  for hostlist in (mechanism_hosts | hostlist_expression) %}
-NodeName={{ hostlist }} AutoDetect={{ mechanism }}
-{%                  endfor %}
-{%              else %}
-{%                  for gres in group.gres %}
-{%                      set gres_name, gres_type, _ = gres.conf.split(':') %}
-{%                      for hostlist in (mechanism_hosts | hostlist_expression) %}
-NodeName={{ hostlist }} Name={{ gres_name }} Type={{ gres_type }} File={{ gres.file | mandatory('The gres configuration dictionary: ' ~  gres ~ ' is missing the file key, but openhpc_gres_autodetect is set to off. The error occured on partition: ' ~  part.name ~ '. Please add the file key or set openhpc_gres_autodetect.') }}
-
-{%                      endfor %}
-{%                  endfor %}
-{%              endif %}
-{% set donehosts = donehosts | union(mechanism_hosts) %}
-{%          endfor %}
-{%      endif %}
-{%  endfor %}
-{% endfor %}
+{% for nodegroup in openhpc_nodegroups %}
+{%      set gres_list = nodegroup.gres | default([]) %}
+{%      set gres_autodetect = nodegroup.gres_autodetect | default('off') %}
+{%      if gres_autodetect | default('off') != 'off'  %}
+NodeName={{ hostlist }} AutoDetect={{ gres_autodetect }}
+{%      else %}
+{%          for gres in gres_list %}
+{%              set gres_name, gres_type, _ = gres.conf.split(':') %}
+{%              set inventory_group_name = openhpc_cluster_name ~ '_' ~ nodegroup.name %}
+{%              set inventory_group_hosts = groups.get(inventory_group_name, []) %}
+{%              for hostlist in (inventory_group_hosts | hostlist_expression) %}
+NodeName={{ hostlist }} Name={{ gres_name }} Type={{ gres_type }} File={{ gres.file }}
+{%              endfor %}{# hostlists #}
+{%          endfor %}{# gres #}
+{%      endif %}{# autodetect #}
+{% endfor %}{# nodegroup #}
diff --git a/templates/slurm.conf.j2 b/templates/slurm.conf.j2
@@ -135,55 +135,57 @@ SlurmdSyslogDebug=info
 #SlurmSchedLogFile=
 #SlurmSchedLogLevel=
 #DebugFlags=
-#
-#
-# POWER SAVE SUPPORT FOR IDLE NODES - NOT SUPPORTED IN THIS APPLIANCE VERSION
 
 # LOGIN-ONLY NODES
 # Define slurmd nodes not in partitions for login-only nodes in "configless" mode:
 {%if openhpc_login_only_nodes %}{% for node in groups[openhpc_login_only_nodes] %}
 NodeName={{ node }}
 {% endfor %}{% endif %}
 
-# COMPUTE NODES
-# OpenHPC default configuration
 PropagateResourceLimitsExcept=MEMLOCK
 Epilog=/etc/slurm/slurm.epilog.clean
-{% set donehosts = [] %}
-{% for part in openhpc_slurm_partitions %}
-    {% set nodelist = [] %}
-    {% for group in part.get('groups', [part]) %}
-        {% set group_name = group.cluster_name|default(openhpc_cluster_name) ~ '_' ~ group.name %}
-# openhpc_slurm_partitions group: {{ group_name }}
-        {% set inventory_group_hosts = groups.get(group_name, []) %}
-        {% if inventory_group_hosts | length > 0 %}
-            {% set play_group_hosts = inventory_group_hosts | intersect (play_hosts) %}
-            {% set first_host = play_group_hosts | first | mandatory('Group "' ~ group_name ~ '" contains no hosts in this play - was --limit used?') %}
-            {% set first_host_hv = hostvars[first_host] %}
-            {% set ram_mb = (first_host_hv['ansible_memory_mb']['real']['total'] * (group.ram_multiplier | default(openhpc_ram_multiplier))) | int %}
-            {% for hostlist in (inventory_group_hosts | hostlist_expression) %}
-        {% set gres = ' Gres=%s' % (','.join(group.gres | map(attribute='conf') )) if 'gres' in group else '' %}
-            {% if hostlist not in donehosts %}
-NodeName={{ hostlist }} State=UNKNOWN RealMemory={{ group.get('ram_mb', ram_mb) }} Sockets={{first_host_hv['ansible_processor_count']}} CoresPerSocket={{ first_host_hv['ansible_processor_cores'] }} ThreadsPerCore={{ first_host_hv['ansible_processor_threads_per_core'] }}{{ gres }}
-            {% endif %}
-                {% set _ = nodelist.append(hostlist) %}
-                {% set _ = donehosts.append(hostlist) %}
-            {% endfor %}{# nodes #}
-        {% endif %}{# inventory_group_hosts #}
-        {% for extra_node_defn in group.get('extra_nodes', []) %}
-{{ extra_node_defn.items() | map('join', '=') | join(' ') }}
-            {% set _ = nodelist.append(extra_node_defn['NodeName']) %}
-        {% endfor %}
-    {% endfor %}{# group #}
-{% if not nodelist %}{# empty partition #}
-{%  set nodelist = ['""'] %}
-{% endif %}
-PartitionName={{part.name}} Default={{ part.get('default', 'YES') }} MaxTime={{ part.get('maxtime', openhpc_job_maxtime) }} State=UP Nodes={{ nodelist | join(',') }} {{ part.partition_params | default({}) | dict2parameters }}
-{% endfor %}{# partitions #}
+
+# COMPUTE NODES
+# OpenHPC default configuration
+{% for nodegroup in openhpc_nodegroups %}
+{%  set inventory_group_name = openhpc_cluster_name ~ '_' ~ nodegroup.name %}
+{%  set inventory_group_hosts = groups.get(inventory_group_name, []) %}
+{%      if inventory_group_hosts | length > 0 %}
+{%          set play_group_hosts = inventory_group_hosts | intersect (play_hosts) %}
+{%          set first_host = play_group_hosts | first | mandatory('Inventory group "' ~ inventory_group_name ~ '" contains no hosts in this play - was --limit used?') %}
+{%          set first_host_hv = hostvars[first_host] %}
+{%          set ram_mb = (first_host_hv['ansible_memory_mb']['real']['total'] * (nodegroup.ram_multiplier | default(openhpc_ram_multiplier))) | int %}
+{%          set hostlists = (inventory_group_hosts | hostlist_expression) %}{# hosts in inventory group aren't necessarily a single hostlist expression #}
+{%              for hostlist in  hostlists %}
+NodeName={{ hostlist }} {{ '' -}}
+    State=UNKNOWN {{ '' -}}
+    RealMemory={{ nodegroup.ram_mb | default(ram_mb) }} {{ '' -}}
+    Sockets={{first_host_hv['ansible_processor_count'] }} {{ '' -}}
+    CoresPerSocket={{ first_host_hv['ansible_processor_cores'] }} {{ '' -}}
+    ThreadsPerCore={{ first_host_hv['ansible_processor_threads_per_core'] }} {{ '' -}}
+    {{ nodegroup.node_params | default({}) | dict2parameters }} {{ '' -}}
+    {% if 'gres' in nodegroup %}Gres={{ ','.join(nodegroup.gres | map(attribute='conf')) }}{% endif %}
+{%              endfor %}{# hostlists #}
+{%      endif %}{# 1 or more hosts in inventory #}
+
+NodeSet={{ nodegroup.name }} Nodes={{ ','.join(hostlists | default(['""'])) }}{# no support for creating nodesets by Feature #}
+
+{% endfor %}
 
 # Define a non-existent node, in no partition, so that slurmctld starts even with all partitions empty
 NodeName=nonesuch
 
+# PARTITIONS
+{% for partition in openhpc_partitions %}
+PartitionName={{partition.name}} {{ '' -}}
+    Default={{ partition.get('default', 'YES') }} {{ '' -}}
+    MaxTime={{ partition.get('maxtime', openhpc_job_maxtime) }} {{ '' -}}
+    State=UP  {{ '' -}}
+    Nodes={{ partition.get('groups', [partition.name]) | join(',') }} {{ '' -}}
+    {{ partition.partition_params | default({}) | dict2parameters }}
+{% endfor %}{# openhpc_partitions #}
+
 {% if openhpc_slurm_configless | bool %}SlurmctldParameters=enable_configless{% endif %}
 
+
 ReturnToService=2
