fix hpctests to work with root-squashed /home

Author: sjpb

ansible/roles/hpctests/README.md

@@ -22,8 +22,10 @@ Requirements
 
 Role Variables
 --------------
-
-- `hpctests_rootdir`: Required. Path to root of test directory tree, which must be on a r/w filesystem shared to all cluster nodes under test. The last directory component will be created.
+- `hpctests_user`: Optional. User to run jobs as. Default is `ansible_user`.
+- `hpctests_rootdir`: Optional. Path to root of test directory tree. This must
+  be a r/w filesystem shared to all cluster nodes under test. Default is
+  `/home/{{ hpctests_user }}/hpctests`. **NB:** Do not use `~` in this path.
 - `hpctests_partition`: Optional. Name of partition to use, otherwise default partition is used.
 - `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the selected partition are used.
 - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use). Alternatively a mapping of partition name (as `hpctests_partition`) to device/interface can be used. For partitions not defined in the mapping the default of `all` is used.

ansible/roles/hpctests/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
-hpctests_rootdir:
+hpctests_user: "{{ ansible_user }}"
+hpctests_rootdir: "/home/{{ hpctests_user }}/hpctests"
 hpctests_pre_cmd: ''
 hpctests_pingmatrix_modules: [gnu12 openmpi4]
 hpctests_pingpong_modules: [gnu12 openmpi4 imb]

ansible/roles/hpctests/library/plot_nxnlatbw.py

@@ -5,7 +5,7 @@
 # Apache 2 License
 
 from ansible.module_utils.basic import AnsibleModule
-import json
+import json, os
 
 ANSIBLE_METADATA = {
     "metadata_version": "0.1",
@@ -109,8 +109,8 @@ def run_module():
     module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
     result = {"changed": False}
 
-    src = module.params["src"]
-    dest = module.params["dest"]
+    src = os.path.expanduser(module.params["src"])
+    dest = os.path.expanduser(module.params["dest"])
     nodes = module.params["nodes"]
     if nodes is not None:
         nodes = nodes.split(',')

ansible/roles/hpctests/tasks/build-hpl.yml

@@ -52,7 +52,6 @@
 
 - name: Build HPL executable
   shell:
-    cmd: "sbatch --wait hpl-build-{{ hpctests_hpl_arch }}.sh"
+    cmd: "bash -l -c 'sbatch --wait hpl-build-{{ hpctests_hpl_arch }}.sh'" # need login shell for module command
     chdir: "{{ hpctests_hpl_srcdir }}"
     creates: "bin/{{ hpctests_hpl_arch }}/xhpl"
-  become: no

ansible/roles/hpctests/tasks/hpl-solo.yml

@@ -80,8 +80,7 @@
     cmd: "rm -f {{ hpctests_rootdir }}/hpl-solo/hpl-solo.sh.*.out"
 
 - name: Run hpl-solo
-  shell: sbatch --wait hpl-solo.sh
-  become: no
+  shell: bash -l -c 'sbatch --wait hpl-solo.sh' # need login shell for module command
   args:
     chdir: "{{ hpctests_rootdir }}/hpl-solo"
   async: "{{ 20 * 60 }}" # wait for up to 20 minutes

ansible/roles/hpctests/tasks/main.yml

@@ -1,28 +1,38 @@
 - name: setup
   block:
-    - include: setup.yml
+    - include_tasks: setup.yml
+  become: true
+  become_user: "{{ hpctests_user }}"
   tags: always
 
 - name: pingpong
   block:
-    - include: pingpong.yml
+    - include_tasks: pingpong.yml
       when: hpctests_computes.stdout_lines | length > 1
+  become: true
+  become_user: "{{ hpctests_user }}"
   tags: pingpong
 
 - name: pingmatrix
   block:
-    - include: pingmatrix.yml
+    - include_tasks: pingmatrix.yml
       when: hpctests_computes.stdout_lines | length > 1
+  become: true
+  become_user: "{{ hpctests_user }}"
   tags: pingmatrix
 
 - name: build HPL
   block:
-    - include: build-hpl.yml
+    - include_tasks: build-hpl.yml
+  become: true
+  become_user: "{{ hpctests_user }}"
   tags:
     - hpl-solo
 
 - name: run HPL on individual nodes
   block:
-    - include: hpl-solo.yml
+    - include_tasks: hpl-solo.yml
+  become: true
+  become_user: "{{ hpctests_user }}"
   tags:
     - hpl-solo

ansible/roles/hpctests/tasks/pingmatrix.yml

@@ -5,14 +5,6 @@
     path: "{{ hpctests_rootdir }}/pingmatrix"
     state: directory
 
-- name: Precreate files to workaround selinux context issues on NFS mounts
-  file:
-    path: "{{ hpctests_rootdir }}/pingmatrix/{{ item }}"
-    state: touch
-  loop:
-    - mpi_nxnlatbw.c
-    - pingmatrix.sh
-
 - name: Copy source
   copy:
     src: mpi_nxnlatbw.c
@@ -24,7 +16,7 @@
     dest: "{{ hpctests_rootdir }}/pingmatrix/pingmatrix.sh"
 
 - name: Run ping matrix
-  shell: sbatch --wait pingmatrix.sh
+  shell: bash -l -c 'sbatch --wait pingmatrix.sh' # need login shell for module command
   args:
     chdir: "{{ hpctests_rootdir }}/pingmatrix"
   register: hpctests_pingmatrix_sbatch

ansible/roles/hpctests/tasks/pingpong.yml

@@ -5,13 +5,6 @@
     path: "{{ hpctests_rootdir }}/pingpong"
     state: directory
 
-- name: Precreate files to workaround selinux context issues on NFS mounts
-  file:
-    path: "{{ hpctests_rootdir }}/pingpong/{{ item }}"
-    state: touch
-  loop:
-    - pingpong.sh
-
 - name: Create sbatch script
   template:
     src: pingpong.sh.j2
@@ -20,8 +13,7 @@
 - name: Run pingpong
   block:
     - name: Submit jobscript
-      shell: sbatch --wait pingpong.sh
-      become: no
+      shell: bash -l -c 'sbatch --wait pingpong.sh' # need login shell for module command
       args:
         chdir: "{{ hpctests_rootdir }}/pingpong"
       register: hpctests_pingpong_sbatch
@@ -54,18 +46,21 @@
     path: "{{ _pingpong_local_output }}"
   register: hpctests_pingpong_out
   delegate_to: localhost
+  become: false
 
 - name: Read nodes used
   shell: "grep 'SLURM_JOB_NODELIST:' {{ _pingpong_local_output }}"
   register: hpctests_pingpong_run_nodes
   delegate_to: localhost
+  become: false
 
 - name: Plot image
   shell:
     cmd: "python {{ role_path }}/files/plot_imb_pingpong.py {{ _pingpong_local_output }}"
     creates: "{{ _pingpong_local_output | dirname }}/latency.png"
   register: _pingpong_plot
   delegate_to: localhost
+  become: false
   when: hpctests_pingpong_plot | bool
 
 - debug:

ansible/roles/hpctests/tasks/setup.yml

@@ -25,9 +25,8 @@
   file:
     path: "{{ hpctests_rootdir }}"
     state: directory
-    owner: "{{ ansible_user }}"
-    group: "{{ ansible_user }}"
-  become: true
+    owner: "{{ hpctests_user }}"
+    group: "{{ hpctests_user }}"
 
 - name: Set fact for UCX_NET_DEVICES
   set_fact:

environments/.stackhpc/inventory/group_vars/all/hpctests.yml

@@ -0,0 +1 @@
+hpctests_user: demo_user