Merge branch 'master' into UES-1279

Author: Vasileios Karakasis

cscs-checks/apps/gromacs/gromacs_check.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import contextlib
-import itertools
 import os
 
 import reframe as rfm
@@ -100,27 +99,37 @@ def __init__(self, scale, variant):
 class GromacsCPUCheck(GromacsBaseCheck):
     def __init__(self, scale, variant):
         super().__init__('md.log')
-        self.valid_systems = ['daint:mc']
+        self.valid_systems = ['daint:mc', 'eiger:mc']
         self.descr = 'GROMACS CPU check'
         self.executable_opts = ['mdrun', '-dlb yes', '-ntomp 1', '-npme -1',
                                 '-nb cpu', '-s herflat.tpr']
 
         if scale == 'small':
             self.valid_systems += ['dom:mc']
-            self.num_tasks = 216
-            self.num_tasks_per_node = 36
+            if (self.current_system.name in ['daint', 'dom']):
+                self.num_tasks = 216
+                self.num_tasks_per_node = 36
+            elif (self.current_system.name in ['eiger']):
+                self.num_tasks = 768
+                self.num_tasks_per_node = 128
         else:
-            self.num_tasks = 576
-            self.num_tasks_per_node = 36
+            if (self.current_system.name in ['daint', 'dom']):
+                self.num_tasks = 576
+                self.num_tasks_per_node = 36
+            elif (self.current_system.name in ['eiger']):
+                self.num_tasks = 2048
+                self.num_tasks_per_node = 128
 
         references = {
             'prod': {
                 'small': {
                     'dom:mc': {'perf': (40.0, -0.05, None, 'ns/day')},
-                    'daint:mc': {'perf': (38.8, -0.10, None, 'ns/day')}
+                    'daint:mc': {'perf': (38.8, -0.10, None, 'ns/day')},
+                    'eiger:mc': {'perf': (103.00, -0.10, None, 'ns/day')}
                 },
                 'large': {
-                    'daint:mc': {'perf': (68.0, -0.20, None, 'ns/day')}
+                    'daint:mc': {'perf': (68.0, -0.20, None, 'ns/day')},
+                    'eiger:mc': {'perf': (146.00, -0.20, None, 'ns/day')}
                 }
             },
         }

reframe/core/logging.py

@@ -247,8 +247,10 @@ def _create_file_handler(site_config, config_prefix):
 
 
 def _create_filelog_handler(site_config, config_prefix):
-    basedir = os.path.abspath(site_config.get(f'{config_prefix}/basedir'))
-    prefix  = site_config.get(f'{config_prefix}/prefix')
+    basedir = os.path.abspath(
+        osext.expandvars(site_config.get(f'{config_prefix}/basedir'))
+    )
+    prefix  = osext.expandvars(site_config.get(f'{config_prefix}/prefix'))
     filename_patt = os.path.join(basedir, prefix)
     append = site_config.get(f'{config_prefix}/append')
     return MultiFileHandler(filename_patt, mode='a+' if append else 'w+')

reframe/core/modules.py

@@ -744,6 +744,7 @@ def __init__(self):
                 (version, self.MIN_VERSION))
 
         self._version = version
+        self._extra_module_paths = []
 
     def name(self):
         return 'tmod4'
@@ -770,6 +771,15 @@ def _execute(self, cmd, *args):
     def load_module(self, module):
         if module.collection:
             self.execute('restore', str(module))
+
+            # Here the module search path removal/addition is repeated since
+            # 'restore' discards previous module path manipulations
+            for op, mp in self._extra_module_paths:
+                if op == '+':
+                    super().searchpath_add(mp)
+                else:
+                    super().searchpath_remove(mp)
+
             return []
         else:
             return super().load_module(module)
@@ -792,7 +802,15 @@ def conflicted_modules(self, module):
 
     def emit_load_instr(self, module):
         if module.collection:
-            return f'module restore {module}'
+            cmds = [f'module restore {module}']
+
+            # Here we append module searchpath removal/addition commands
+            # since 'restore' discards previous module path manipulations
+            for op, mp in self._extra_module_paths:
+                operation = 'use' if op == '+' else 'unuse'
+                cmds += [f'module {operation} {mp}']
+
+            return '\n'.join(cmds)
 
         return super().emit_load_instr(module)
 
@@ -802,6 +820,18 @@ def emit_unload_instr(self, module):
 
         return super().emit_unload_instr(module)
 
+    def searchpath_add(self, *dirs):
+        if dirs:
+            self._extra_module_paths += [('+', mp) for mp in dirs]
+
+        super().searchpath_add(*dirs)
+
+    def searchpath_remove(self, *dirs):
+        if dirs:
+            self._extra_module_paths += [('-', mp) for mp in dirs]
+
+        super().searchpath_remove(*dirs)
+
 
 class LModImpl(TMod4Impl):
     '''Module system for Lmod (Tcl/Lua).'''
@@ -836,6 +866,8 @@ def __init__(self):
             raise ConfigError('Python is not supported by '
                               'this Lmod installation')
 
+        self._extra_module_paths = []
+
     def name(self):
         return 'lmod'
 
@@ -891,20 +923,6 @@ def conflicted_modules(self, module):
 
         return ret
 
-    def load_module(self, module):
-        if module.collection:
-            self.execute('restore', str(module))
-            return []
-        else:
-            return super().load_module(module)
-
-    def unload_module(self, module):
-        if module.collection:
-            # Module collection are not unloaded
-            return
-
-        super().unload_module(module)
-
     def unload_all(self):
         # Currently, we don't take any provision for sticky modules in Lmod, so
         # we forcefully unload everything.

requirements.txt

@@ -1,6 +1,6 @@
 importlib_metadata==2.0.0
 jsonschema==3.2.0
-pytest==6.1.1
+pytest==6.2.0
 pytest-forked==1.3.0
 pytest-parallel==0.1.0
 coverage==5.3

unittests/test_cli.py

@@ -364,8 +364,22 @@ def test_performance_check_failure(run_reframe, tmp_path, perflogdir):
                           'default' / 'PerformanceFailureCheck.log')
 
 
-def test_performance_report(run_reframe):
+def test_perflogdir_from_env(run_reframe, tmp_path, monkeypatch):
+    monkeypatch.setenv('FOODIR', str(tmp_path / 'perflogs'))
     returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/frontend_checks.py'],
+        more_options=['-t', 'PerformanceFailureCheck'],
+        perflogdir='$FOODIR'
+    )
+    assert returncode == 1
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert os.path.exists(tmp_path / 'perflogs' / 'generic' /
+                          'default' / 'PerformanceFailureCheck.log')
+
+
+def test_performance_report(run_reframe):
+    returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
         more_options=['-t', 'PerformanceFailureCheck', '--performance-report']
     )
@@ -657,6 +671,7 @@ def test_unuse_module_path(run_reframe, user_exec_ctx):
         config_file=fixtures.USER_CONFIG_FILE, action='run',
         system=rt.runtime().system.name
     )
+    ms.searchpath_remove(module_path)
     assert "could not load module 'testmod_foo' correctly" in stdout
     assert 'Traceback' not in stderr
     assert returncode == 0
@@ -673,7 +688,6 @@ def test_use_module_path(run_reframe, user_exec_ctx):
         config_file=fixtures.USER_CONFIG_FILE, action='run',
         system=rt.runtime().system.name
     )
-
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
     assert "could not load module 'testmod_foo' correctly" not in stdout
@@ -694,7 +708,6 @@ def test_overwrite_module_path(run_reframe, user_exec_ctx):
         config_file=fixtures.USER_CONFIG_FILE, action='run',
         system=rt.runtime().system.name
     )
-
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
     assert "could not load module 'testmod_foo' correctly" not in stdout

unittests/test_modules.py

@@ -3,17 +3,13 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import abc
 import os
 import pytest
 
 import reframe.core.environments as env
 import reframe.core.modules as modules
-import reframe.utility as util
-import reframe.utility.osext as osext
 import unittests.fixtures as fixtures
-from reframe.core.exceptions import (ConfigError, EnvironError)
-from reframe.core.runtime import runtime
+from reframe.core.exceptions import ConfigError, EnvironError
 
 
 @pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod'])
@@ -130,6 +126,18 @@ def test_module_load_force_collection(modules_system, module_collection):
     assert modules_system.is_module_loaded('testmod_foo')
 
 
+def test_module_load_collection_searchpath(modules_system, tmpdir,
+                                           module_collection):
+    p1 = str(tmpdir.mkdir('path1'))
+    p2 = str(tmpdir.mkdir('path2'))
+    modules_system.searchpath_add(p1)
+    modules_system.searchpath_add(p2)
+    modules_system.searchpath_remove(p1)
+    modules_system.load_module(module_collection, collection=True)
+    assert p1 not in modules_system.searchpath
+    assert p2 in modules_system.searchpath
+
+
 def test_module_unload_all(modules_system):
     if modules_system.name == 'nomod':
         modules_system.unload_all()

unittests/test_utility.py

@@ -1414,7 +1414,8 @@ def modules_system(user_exec_ctx, monkeypatch):
 
     ms = rt.runtime().system.modules_system
     ms.searchpath_add(fixtures.TEST_MODULES)
-    return ms
+    yield ms
+    ms.searchpath_remove(fixtures.TEST_MODULES)
 
 
 def test_find_modules(modules_system):