Merge pull request #217 from stackhpc/feature/multipart-hpctests2

Support multiple partitions in MPI tests

Author: sjpb

ansible/adhoc/hpctests.yml

@@ -4,7 +4,7 @@
 
 ---
 
-- hosts: hpctests[0] # TODO: might want to make which node is used selectable?
+- hosts: login[0] # TODO: might want to make which node is used selectable?
   become: false
   gather_facts: false
   tasks:

ansible/ci/test_reimage.yml

@@ -45,10 +45,10 @@
   gather_facts: no
   tags: reimage_compute
   tasks:
-    # TODO: This is specific to smslabs/arcus environment config - could generalise to all compute nodes
+    # TODO: This is specific to arcus environment config - could generalise to all compute nodes
     - name: Request compute node rebuild via Slurm
       shell:
-        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1]
+        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-3]
       become: yes
 
     - name: Check compute node rebuild completed

ansible/roles/hpctests/README.md

@@ -24,7 +24,8 @@ Role Variables
 --------------
 
 - `hpctests_rootdir`: Required. Path to root of test directory tree, which must be on a r/w filesystem shared to all cluster nodes under test. The last directory component will be created.
-- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the default partition are used. Note nodes selected **must** be in the default partition.
+- `hpctests_partition`: Optional. Name of partition to use, otherwise default partition is used.
+- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the selected partition are used.
 - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use).
 - `hpctests_outdir`: Optional. Directory to use for test output on local host. Defaults to `$HOME/hpctests` (for local user).
 - `hpctests_hpl_NB`: Optional, default 192. The HPL block size "NB" - for Intel CPUs see [here](https://software.intel.com/content/www/us/en/develop/documentation/onemkl-linux-developer-guide/top/intel-oneapi-math-kernel-library-benchmarks/intel-distribution-for-linpack-benchmark/configuring-parameters.html).

ansible/roles/hpctests/defaults/main.yml

@@ -10,3 +10,4 @@ hpctests_hpl_NB: 192
 hpctests_hpl_mem_frac: 0.8
 hpctests_hpl_arch: linux64
 #hpctests_nodes:
+#hpctests_partition:

ansible/roles/hpctests/library/slurm_node_info.py

@@ -23,7 +23,7 @@
 options
     nodes:
         description:
-            - Slurm nodenames for which information is required. These must be homogenous.
+            - Slurm nodenames for which information is required.
         required: true
         type: list
 requirements:
@@ -56,7 +56,6 @@ def run_module():
     print(values)
     for ix, param in enumerate(params):
         info[param] = [nodeinfo[ix].strip() for nodeinfo in values if nodeinfo[nodelist_ix].strip() in module.params['nodes']]
-        # info[param] = [nodeinfo[nodelist_ix] for nodeinfo in values]
     result['info'] = info
 
     module.exit_json(**result)

ansible/roles/hpctests/tasks/hpl-solo.yml

@@ -42,8 +42,8 @@
 - debug:
     msg: "Using {{ hpctests_hplsolo_ntasks }} process per node with P={{ hpctests_hplsolo_pq.grid.P }}, Q={{ hpctests_hplsolo_pq.grid.Q }} targeting {{ (hpctests_hpl_mem_frac | float)  * 100 }}% of {{ hpctests_nodeinfo.info['MEMORY'][0] }} MB memory per node, block size (NB) = {{ hpctests_hpl_NB }}, problem size (N) = {{ hpctests_hplsolo_N }}"
 
-- name: Get all nodes
-  shell: "sinfo --Node --noheader --format %N" # TODO: assumes only one partition, although actually excluding nodes not in the default partition should be fine.
+- name: Get all nodes in partition
+  shell: "sinfo --Node --noheader --format %N --partition={{ hpctests_partition }}"
   register: all_nodes
   changed_when: false
 
@@ -74,6 +74,11 @@
   vars:
     hpctests_hplsolo_ntasks: 2 # TODO: FIXME
 
+- name: Remove previous outputs
+  # As depending on the number of nodes there will be different numbers of output files for different partitions so won't all get overwritten
+  shell:
+    cmd: "rm -f {{ hpctests_rootdir }}/hpl-solo/hpl-solo.sh.*.out"
+
 - name: Run hpl-solo
   shell: sbatch --wait hpl-solo.sh
   become: no
@@ -111,10 +116,11 @@
   tags: postpro
   debug:
     msg: |
-      Summary for hpl-solo ({{ hpctests_computes.stdout_lines | length }} nodes) job {{ hpctests_hplsolo_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}:
+      Summary for hpl-solo on {{ hpctests_computes.stdout_lines | length }} nodes in '{{ hpctests_partition }}' partition, job ID {{ hpctests_hplsolo_sbatch.stdout.split()[-1] }}, device '{{ hpctests_ucx_net_devices }}':
+
         Max:  {{ perf.stdout_lines | map('float') | max }} gflops
         Min:  {{ perf.stdout_lines | map('float') | min }} gflops
-        Mean: {{ (perf.stdout_lines | map('float') | sum) /  (hpctests_computes.stdout_lines | length) }} gflops
+        Mean: {{ (perf.stdout_lines | map('float') | sum) / (hpctests_computes.stdout_lines | length) }} gflops
 
       Individual node results (gflops):
       {{ dict(hpctests_computes.stdout_lines | zip(perf.stdout_lines | map('float') )) | to_nice_yaml }}

ansible/roles/hpctests/tasks/pingmatrix.yml

@@ -61,6 +61,8 @@
 - name: Summarise results
   debug:
     msg: |
-      Summary for pingmatrix (pairwise on {{ slurm_names.stdout_lines | length }} nodes) job {{ hpctests_pingmatrix_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}:
+      Summary for pingmatrix pairwise over {{ slurm_names.stdout_lines | length }} nodes in '{{ hpctests_partition }}' partition, job ID {{ hpctests_pingmatrix_sbatch.stdout.split()[-1] }}, device '{{ hpctests_ucx_net_devices }}':
+      
       {{ nxnlatbw['stats'] | to_nice_yaml }}
+      
       Tabular output on ansible control host at {{ hpctests_outdir }}/pingmatrix.html

ansible/roles/hpctests/tasks/pingpong.yml

@@ -55,10 +55,11 @@
 
 - debug:
     msg: |
-      Summary for pingpong (2x scheduler-selected nodes) job {{ _pingpong_jobid }} (using interface {{ hpctests_ucx_net_devices }}):
-      nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }}
-      zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us
-      max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s)
+      Summary for pingpong using 2x scheduler-selected nodes in '{{ hpctests_partition }}' partition, job ID {{ _pingpong_jobid }}, device '{{ hpctests_ucx_net_devices }}':
+
+      Nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }}
+      Zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us
+      Max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s)
 
       See plot on localhost:
       {{ _pingpong_plot.stdout }}

ansible/roles/hpctests/tasks/setup.yml

@@ -1,7 +1,17 @@
 ---
 
+- name: Get partition information
+  shell: "sinfo --format %P --noheader"
+  register: _sinfo_partitions
+  changed_when: false
+
+- name: Select default partition if hpctests_partition not given
+  set_fact:
+    hpctests_partition: "{{ _sinfo_partitions.stdout_lines  | select('contains', '*') | first | trim('*') }}"
+  when: hpctests_partition is not defined
+
 - name: Get info about compute nodes
-  shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --format %N"
+  shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --partition {{hpctests_partition}} --format %N"
   register: hpctests_computes
   changed_when: false
   failed_when: hpctests_computes.rc != 0

ansible/roles/hpctests/templates/hpl-build.sh.j2

@@ -4,6 +4,7 @@
 #SBATCH --output=%x.%a.out
 #SBATCH --error=%x.%a.out
 #SBATCH --exclusive
+#SBATCH --partition={{ hpctests_partition }}
 {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_computes.stdout_lines[0] }}{% endif %}
 
 echo HPL arch: {{ hpctests_hpl_arch }}