add top level topology override + gate plugin on group being enabled

Author: wtripp180901

ansible/roles/topology/README.md

@@ -12,4 +12,17 @@ Role Variables
 
 - `topology_topology_nodes:`: Required list of strs. List of inventory hostnames of nodes to include in topology tree. Must be set to include all compute nodes in Slurm cluster. Default `[]`.
 - `topology_conf_template`: Optional str. Path to Jinja2 template of topology.conf file. Default
-  `templates/topology.conf.j2`
+  `templates/topology.conf.j2`
+- `topology_above_rack_topology`: Optionally multiline str. Used to define topology above racks/AZs if 
+   you wish to partition racks further under different logical switches. New switches above should be
+   defined as [SwitchName lines](https://slurm.schedmd.com/topology.html#hierarchical) referencing 
+   rack Availability Zones under that switch in their `Switches fields`. These switches must themselves
+   be under a top level switch. e.g
+   ```
+   topology_above_rack_topology: |
+     SwitchName=rack-group-1 Switches=rack-az-1,rack-az-2
+     SwitchName=rack-group-2 Switches=rack-az-3,rack-az-4
+     SwitchName=top-level Switches=rack-group-1,rack-group-2
+   ```
+   Defaults to an empty string, which causes all AZs to be put under a
+   single top level switch.

ansible/roles/topology/defaults/main.yml

@@ -3,3 +3,6 @@ topology_topology_nodes: []
 
 # Override to use custom topology.conf template
 topology_conf_template: templates/topology.conf.j2
+
+topology_above_rack_topology: ""
+

ansible/roles/topology/templates/topology.conf.j2

@@ -6,4 +6,8 @@ SwitchName={{ instance_host }} Nodes={{ _topology.topology[az][instance_host] |
 {% endfor %}
 SwitchName={{ az }} Switches={{ _topology.topology[az].keys() | join(",") }}
 {% endfor %}
+{% if topology_above_rack_topology == '' %}
 SwitchName=master Switches={{ _topology.topology.keys() | join(",") }}
+{% else %}
+{{ topology_above_rack_topology }}
+{% endif %}

environments/common/inventory/group_vars/all/openhpc.yml

@@ -57,7 +57,7 @@ openhpc_config_default:
     - enable_configless
   TaskPlugin: task/cgroup,task/affinity
   ReturnToService: 2 # workaround for templating bug TODO: Remove once on stackhpc.openhpc v1.2.0
-  TopologyPlugin: topology/tree
+  TopologyPlugin: "topology/{{ 'tree' if (topology_topology_nodes | length) > 0 else 'flat' }}"
 
 # default additional slurm.conf parameters when "rebuild" enabled:
 openhpc_config_rebuild: