support a subset of gres

Author: sjpb

README.md

@@ -58,8 +58,14 @@ each list element:
   * `name`: The name of the nodes within this group.
   * `cluster_name`: Optional.  An override for the top-level definition `openhpc_cluster_name`.
   * `extra_nodes`: Optional. A list of additional node definitions, e.g. for nodes in this group/partition not controlled by this role. Each item should be a dict, with keys/values as per the ["NODE CONFIGURATION"](https://slurm.schedmd.com/slurm.conf.html#lbAE) docs for slurm.conf. Note the key `NodeName` must be first.
-  * `ram_mb`: Optional.  The physical RAM available in each server of this group ([slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `RealMemory`) in MiB. This is set using ansible facts if not defined, equivalent to `free --mebi` total * `openhpc_ram_multiplier`.
+  * `ram_mb`: Optional.  The physical RAM available in each node of this group ([slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `RealMemory`) in MiB. This is set using ansible facts if not defined, equivalent to `free --mebi` total * `openhpc_ram_multiplier`.
   * `ram_multiplier`: Optional.  An override for the top-level definition `openhpc_ram_multiplier`. Has no effect if `ram_mb` is set.
+  * `gres`: Optional. List of dicts defining [generic resources](https://slurm.schedmd.com/gres.html). Each dict must define:
+      - `conf`: A string with the [resource specification](https://slurm.schedmd.com/slurm.conf.html#OPT_Gres_1) but requiring the format `<name>:<type>:<number>`, e.g. `gpu:A100:2`. Note the `type` is an arbitrary string.
+      - `file`: A string with the [File](https://slurm.schedmd.com/gres.conf.html#OPT_File) (path to device(s)) for this resource, e.g. `/dev/nvidia[0-1]` for the above example.
+
+    Note [GresTypes](https://slurm.schedmd.com/slurm.conf.html#OPT_GresTypes) must be set in `openhpc_config` if this is used.
+
 * `default`: Optional.  A boolean flag for whether this partion is the default.  Valid settings are `YES` and `NO`.
 * `maxtime`: Optional.  A partition-specific time limit following the format of [slurm.conf](https://slurm.schedmd.com/slurm.conf.html) parameter `MaxTime`.  The default value is
   given by `openhpc_job_maxtime`. The value should be quoted to avoid Ansible conversions.

defaults/main.yml

@@ -13,6 +13,7 @@ openhpc_resume_timeout: 300
 openhpc_retry_delay: 10
 openhpc_job_maxtime: '60-0' # quote this to avoid ansible converting some formats to seconds, which is interpreted as minutes by Slurm
 openhpc_config: "{{ openhpc_extra_config | default({}) }}"
+openhpc_gres_template: gres.conf.j2
 openhpc_slurm_configless: "{{ 'enable_configless' in openhpc_config.get('SlurmctldParameters', []) }}"
 
 openhpc_state_save_location: /var/spool/slurm

tasks/runtime.yml

@@ -108,7 +108,7 @@
   changed_when: false # so molecule doesn't fail
   become: no
 
-- name: Create slurm.conf on control node
+- name: Create slurm.conf
   copy:
     src: "{{ _slurm_conf_tmpfile.path }}"
     dest: /etc/slurm/slurm.conf
@@ -121,6 +121,19 @@
   register: ohpc_slurm_conf
   # NB uses restart rather than reload as number of nodes might have changed
 
+- name: Create gres.conf
+  template:
+    src: "{{ openhpc_gres_template }}"
+    dest: /etc/slurm/gres.conf
+    mode: "0600"
+    owner: slurm
+    group: slurm
+  when: openhpc_enable.control | default(false) or not openhpc_slurm_configless
+  notify:
+    - Restart slurmctld service
+  register: ohpc_gres_conf
+  # NB uses restart rather than reload as this is needed in some cases
+
 - name: Remove local tempfile for slurm.conf templating
   ansible.builtin.file:
     path: "{{ _slurm_conf_tmpfile.path }}"
@@ -136,7 +149,7 @@
   changed_when: true
   when:
     - openhpc_slurm_control_host in ansible_play_hosts
-    - hostvars[openhpc_slurm_control_host].ohpc_slurm_conf.changed # noqa no-handler
+    - hostvars[openhpc_slurm_control_host].ohpc_slurm_conf.changed or hostvars[openhpc_slurm_control_host].ohpc_gres_conf.changed # noqa no-handler
   notify:
     - Restart slurmd service
 

templates/gres.conf.j2

@@ -0,0 +1,16 @@
+AutoDetect=off
+{% for part in openhpc_slurm_partitions %}
+{%  set nodelist = [] %}
+{%  for group in part.get('groups', [part]) %}
+{%      if 'gres' in group %}
+{%          for gres in group.gres %}
+{%              set gres_name, gres_type, _ = gres.conf.split(':') %}
+{%              set group_name = group.cluster_name|default(openhpc_cluster_name) ~ '_' ~ group.name %}
+{%              set inventory_group_hosts = groups.get(group_name, []) %}
+{%              for hostlist in (inventory_group_hosts | hostlist_expression) %}
+NodeName={{ hostlist }} Name={{ gres_name }} Type={{ gres_type }} File={{ gres.file }}
+{%              endfor %}
+{%          endfor %}
+{%      endif %}
+{%  endfor %}
+{% endfor %}

templates/slurm.conf.j2

@@ -118,7 +118,9 @@ Epilog=/etc/slurm/slurm.epilog.clean
             {% set first_host_hv = hostvars[first_host] %}
             {% set ram_mb = (first_host_hv['ansible_memory_mb']['real']['total'] * (group.ram_multiplier | default(openhpc_ram_multiplier))) | int %}
             {% for hostlist in (inventory_group_hosts | hostlist_expression) %}
-NodeName={{ hostlist }} State=UNKNOWN RealMemory={{ group.get('ram_mb', ram_mb) }} Sockets={{first_host_hv['ansible_processor_count']}} CoresPerSocket={{ first_host_hv['ansible_processor_cores'] }} ThreadsPerCore={{ first_host_hv['ansible_processor_threads_per_core'] }}
+        {% set gres = ' Gres=%s' % (','.join(group.gres | map(attribute='conf') )) if 'gres' in group else '' %}
+
+NodeName={{ hostlist }} State=UNKNOWN RealMemory={{ group.get('ram_mb', ram_mb) }} Sockets={{first_host_hv['ansible_processor_count']}} CoresPerSocket={{ first_host_hv['ansible_processor_cores'] }} ThreadsPerCore={{ first_host_hv['ansible_processor_threads_per_core'] }}{{ gres }}
                 {% set _ = nodelist.append(hostlist) %}
             {% endfor %}{# nodes #}
         {% endif %}{# inventory_group_hosts #}