Merge branch 'main' into regression-plugin

Author: ekouts

checks/apps/cp2k/cp2k_uenv.py

@@ -15,11 +15,11 @@
 
 cp2k_references = {
     'md': {
-        'gh200': {'time_run': (69, None, 0.05, 's')},
+        'gh200': {'time_run': (45, None, 0.05, 's')},
         'zen2': {'time_run': (91, None, 0.05, 's')}
     },
     'pbe': {
-        'gh200': {'time_run': (67, None, 0.05, 's')},
+        'gh200': {'time_run': (50, None, 0.05, 's')},
         'zen2': {'time_run': (68, None, 0.05, 's')}
     },
     'rpa': {

checks/apps/cp2k/src/mps-wrapper.sh

@@ -1,19 +1,33 @@
 #!/bin/bash
 
-set -u
+set -eu
 
-export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps
-export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-log
-export CUDA_VISIBLE_DEVICES=$(( SLURM_LOCALID % 4 ))
+export CUDA_DEVICE_MAX_COPY_CONNECTIONS=8
+export CUDA_DEVICE_MAX_CONNECTIONS=8
 
-if [ "${SLURM_LOCALID}" -eq 0 ]; then
-    CUDA_VISIBLE_DEVICES=0,1,2,3 nvidia-cuda-mps-control -d
+export CUDA_MPS_PIPE_DIRECTORY=/tmp/nvidia-mps-$((SLURM_LOCALID % 4))
+export CUDA_MPS_LOG_DIRECTORY=/tmp/nvidia-log-$((SLURM_LOCALID % 4))-$(id -un)
+
+export HWLOC_KEEP_NVIDIA_GPU_NUMA_NODES=0
+numa_nodes=$(hwloc-calc --physical --intersect NUMAnode $(hwloc-bind --get --taskset)) # do not set CUDA_VISIBLE_DEVICES, enough to set it for the daemon
+
+# Launch MPS from a single rank per GPU
+if [[ $SLURM_LOCALID -lt 4 ]]; then
+    mkdir -p ${CUDA_MPS_PIPE_DIRECTORY}
+    mkdir -p ${CUDA_MPS_LOG_DIRECTORY}
+    CUDA_VISIBLE_DEVICES=$((SLURM_LOCALID % 4)) nvidia-cuda-mps-control -d
 fi
 
-sleep 5
+# Wait for MPS to start
+sleep 1
 
-exec "$@"
+# Run the command
+"$@"
+result=$?
 
-if [ "${SLURM_LOCALID}" -eq 0 ]; then
+# Quit MPS control daemon before exiting
+if [[ $SLURM_LOCALID -lt 4 ]]; then
     echo quit | nvidia-cuda-mps-control
 fi
+
+exit $result

checks/apps/gromacs/gromacs_check.py

@@ -96,16 +96,21 @@ def prepare_build(self):
         )
         self.prebuild_cmds = [f'tar --strip-components=1 -xzf {tarsource}']
         self.build_system.config_opts = [
-            '-DREGRESSIONTEST_DOWNLOAD=ON',
-            '-DGMX_MPI=on',
-            '-DGMX_BUILD_OWN_FFTW=ON',
-            '-DCP2K_USE_SPGLIB=ON',
+            '-DREGRESSIONTEST_DOWNLOAD=OFF',
+            '-DGMX_MPI=ON',
+            '-DGMX_BUILD_OWN_FFTW=OFF',
             '-DGMX_HWLOC=ON',
-            '-DGMX_SIMD=ARM_NEON_ASIMD',
             '-DGMX_INSTALL_NBLIB_API=ON',
         ]
         if self.uarch == 'gh200':
-            self.build_system.config_opts += ['-DGMX_GPU=CUDA']
+            self.build_system.config_opts += [
+                '-DGMX_SIMD=ARM_NEON_ASIMD',
+                '-DGMX_GPU=CUDA'
+            ]
+        elif self.uarch == 'zen2':
+            self.build_system.config_opts += [
+                '-DGMX_SIMD=AUTO',
+            ]
 
     @sanity_function
     def validate_test(self):
@@ -115,7 +120,7 @@ def validate_test(self):
 
 @rfm.simple_test
 class gromacs_run_test(rfm.RunOnlyRegressionTest):
-    executable = './mps-wrapper.sh -- gmx_mpi mdrun -s stmv2.tpr'
+    executable = 'gmx_mpi mdrun -s stmv2.tpr'
     executable_opts = [
         '-dlb no', '-npme 1', '-pin off', '-v', '-noconfout', '-nstlist 300'
     ]
@@ -143,6 +148,8 @@ def prepare_run(self):
         self.executable_opts.append(f'-ntomp {self.num_cpus_per_task}')
 
         if self.uarch == 'gh200':
+            self.executable = './mps-wrapper.sh -- ' + self.executable
+
             self.executable_opts += [
                 '-pme gpu', '-nb gpu', '-update gpu', '-nsteps 10000'
             ]

checks/system/slurm/slurm.py

@@ -13,7 +13,7 @@
 import reframe.utility.osext as osext
 import reframe.utility.sanity as sn
 
-sys.path.append(str(pathlib.Path(__file__).parent / 'mixins'))
+sys.path.append(str(pathlib.Path(__file__).parent.parent.parent / 'mixins'))
 from uenv_slurm_mpi_options import UenvSlurmMpiOptionsMixin  # noqa: E402
 
 
@@ -197,7 +197,8 @@ def assert_found_exceeded_memory(self):
 
 
 @rfm.simple_test
-class MemoryOverconsumptionCheckMPI(SlurmCompiledBaseCheck, UenvSlurmMpiOptionsMixin):
+class MemoryOverconsumptionCheckMPI(SlurmCompiledBaseCheck,
+                                    UenvSlurmMpiOptionsMixin):
     # TODO: maintainers = ['@jgphpc', '@ekouts']
     descr = 'Tests for max allocatable memory'
     valid_systems = ['+remote']
@@ -441,13 +442,19 @@ class SlurmPrologEpilogCheck(rfm.RunOnlyRegressionTest):
     epilog_dir = '/etc/slurm/node_epilog.d/'
     prerun_cmds = [f'ln -s {kafka_logger} ./kafka_logger']
     test_files = []
-    for file in os.listdir(epilog_dir):
-        if os.path.isfile(os.path.join(epilog_dir, file)):
-            test_files.append(os.path.join(epilog_dir, file))
-
-    for file in os.listdir(prolog_dir):
-        if os.path.isfile(os.path.join(prolog_dir, file)):
-            test_files.append(os.path.join(prolog_dir, file))
+    try:
+        for file in os.listdir(epilog_dir):
+            if os.path.isfile(os.path.join(epilog_dir, file)):
+                test_files.append(os.path.join(epilog_dir, file))
+    except PermissionError:
+        pass
+
+    try:
+        for file in os.listdir(prolog_dir):
+            if os.path.isfile(os.path.join(prolog_dir, file)):
+                test_files.append(os.path.join(prolog_dir, file))
+    except PermissionError:
+        pass
 
     test_file = parameter(test_files)
     tags = {'vs-node-validator'}
@@ -544,6 +551,28 @@ def validate(self):
         return sn.assert_not_found(r'\bisolcpus=', self.stdout)
 
 
+@rfm.simple_test
+class SlurmNoUvmPerfAccessCounterMigration(rfm.RunOnlyRegressionTest):
+    valid_systems = ['+remote +scontrol +nvgpu']
+    valid_prog_environs = ['builtin']
+    maintainers = ['msimberg', 'SSA']
+    descr = '''
+    Check that uvm_perf_access_counter_mimc_migration_enable is set to 0
+    as it is buggy in older drivers.
+    '''
+    time_limit = '1m'
+    num_tasks_per_node = 1
+    sourcesdir = None
+    executable = 'cat'
+    executable_opts = [('/sys/module/nvidia_uvm/parameters/'
+                        'uvm_perf_access_counter_mimc_migration_enable')]    
+    tags = {'production', 'maintenance', 'slurm'}
+
+    @sanity_function
+    def validate(self):
+        return sn.assert_found(r'0', self.stdout)
+
+
 @rfm.simple_test
 class SlurmGPUGresTest(SlurmSimpleBaseCheck):
     descr = '''
@@ -572,11 +601,20 @@ def assert_gres_valid(self):
         gpu_count = self.current_partition.select_devices('gpu')[0].num_devices
         part_re = rf'Partitions=\S*{partition_name}'
         gres_re = rf'gres/gpu={gpu_count} '
-        node_count = sn.count(sn.extractall(part_re, self.stdout))
-        gres_count = sn.count(
-            sn.extractall(rf'{part_re}.*{gres_re}', self.stdout))
-        return sn.assert_eq(
-            node_count, gres_count,
-            f'{gres_count}/{node_count} of '
-            f'{partition_name} nodes satisfy {gres_re}'
+        node_re = r'NodeName=(\S+)'
+
+        all_nodes = sn.evaluate(
+            sn.extractall(rf'{node_re}.*{part_re}', self.stdout, 1)
+        )
+        good_nodes = sn.evaluate(
+            sn.extractall(rf'{node_re}.*{part_re}.*{gres_re}',
+                          self.stdout, 1)
+        )
+        bad_nodes = ','.join(sorted(set(all_nodes) - set(good_nodes)))
+
+        return sn.assert_true(
+            len(bad_nodes) == 0,
+            msg=(f'{len(good_nodes)}/{len(all_nodes)} of '
+                 f'{partition_name} nodes satisfy {gres_re}. Bad nodes: '
+                 f'{bad_nodes}')
         )