Merge pull request #1624 from rsarm/feat/slack-load

[feat] Support `spack load` for loading software packages

Author: Vasileios Karakasis

ci-scripts/configs/spack.py

@@ -0,0 +1,70 @@
+# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+#
+# Generic fallback configuration
+#
+
+site_configuration = {
+    'systems': [
+        {
+            'name': 'generic',
+            'descr': 'Generic example system',
+            'hostnames': ['.*'],
+            'modules_system': 'spack',
+            'partitions': [
+                {
+                    'name': 'default',
+                    'scheduler': 'local',
+                    'launcher': 'local',
+                    'environs': ['builtin']
+                }
+            ]
+        },
+    ],
+    'environments': [
+        {
+            'name': 'builtin',
+            'cc': 'cc',
+            'cxx': '',
+            'ftn': ''
+        },
+    ],
+    'logging': [
+        {
+            'handlers': [
+                {
+                    'type': 'stream',
+                    'name': 'stdout',
+                    'level': 'info',
+                    'format': '%(message)s'
+                },
+                {
+                    'type': 'file',
+                    'level': 'debug',
+                    'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s',   # noqa: E501
+                    'append': False
+                }
+            ],
+            'handlers_perflog': [
+                {
+                    'type': 'filelog',
+                    'prefix': '%(check_system)s/%(check_partition)s',
+                    'level': 'info',
+                    'format': (
+                        '%(check_job_completion_time)s|reframe %(version)s|'
+                        '%(check_info)s|jobid=%(check_jobid)s|'
+                        '%(check_perf_var)s=%(check_perf_value)s|'
+                        'ref=%(check_perf_ref)s '
+                        '(l=%(check_perf_lower_thres)s, '
+                        'u=%(check_perf_upper_thres)s)|'
+                        '%(check_perf_unit)s'
+                    ),
+                    'append': True
+                }
+            ]
+        }
+    ],
+}

ci-scripts/dockerfiles/Spack.dockerfile

@@ -0,0 +1,40 @@
+#
+# Execute this from the top-level ReFrame source directory
+#
+
+FROM ubuntu:20.04
+
+ENV TZ=Europe/Zurich
+ENV DEBIAN_FRONTEND=noninteractive
+ENV _SPACK_VER=0.16
+
+# ReFrame user
+RUN useradd -ms /bin/bash rfmuser
+
+# ReFrame requirements
+RUN \
+  apt-get -y update && \
+  apt-get -y install ca-certificates && \
+  update-ca-certificates && \
+  apt-get -y install gcc && \
+  apt-get -y install make && \
+  apt-get -y install git && \
+  apt-get -y install python3 python3-pip
+
+# Install ReFrame from the current directory
+COPY --chown=rfmuser . /home/rfmuser/reframe/
+
+USER rfmuser
+
+# Install Spack
+RUN git clone https://github.com/spack/spack ~/spack && \
+    cd ~/spack && \
+    git checkout releases/v${_SPACK_VER}
+
+ENV BASH_ENV /home/rfmuser/spack/share/spack/setup-env.sh
+
+WORKDIR /home/rfmuser/reframe
+
+RUN ./bootstrap.sh
+
+CMD ["/bin/bash", "-c", "./test_reframe.py --rfm-user-config=ci-scripts/configs/spack.py -v"]

docs/config_reference.rst

@@ -115,8 +115,12 @@ System Configuration
    - ``tmod32``: A synonym of ``tmod``.
    - ``tmod4``: The `new environment modules <http://modules.sourceforge.net/>`__ implementation (versions older than 4.1 are not supported).
    - ``lmod``: The `Lua implementation <https://lmod.readthedocs.io/en/latest/>`__ of the environment modules.
+   - ``spack``: `Spack <https://spack.readthedocs.io/en/latest/>`'s built-in mechanism for managing modules.
    - ``nomod``: This is to denote that no modules system is used by this system.
 
+   .. versionadded:: 3.4
+      The ``spack`` backend is added.
+
 .. js:attribute:: .systems[].modules
 
    :required: No

reframe/core/modules.py

@@ -109,6 +109,8 @@ def create(cls, modules_kind=None):
             return ModulesSystem(TMod4Impl())
         elif modules_kind == 'lmod':
             return ModulesSystem(LModImpl())
+        elif modules_kind == 'spack':
+            return ModulesSystem(SpackImpl())
         else:
             raise ConfigError('unknown module system: %s' % modules_kind)
 
@@ -983,3 +985,88 @@ def emit_load_instr(self, module):
 
     def emit_unload_instr(self, module):
         return ''
+
+
+class SpackImpl(ModulesSystemImpl):
+    '''Backend for Spack's modules system emulation.
+
+    This backend implements :func:`load_module`, :func:`unload_module` as well
+    as the searchpath methods as no-ops, since Spack does not offer any Python
+    bindings for its emulation.
+
+    '''
+
+    def __init__(self):
+        # Try to figure out if we are indeed using the TCL version
+        try:
+            completed = osext.run_command('spack -V')
+        except OSError as e:
+            raise ConfigError(
+                'could not find a sane Spack installation') from e
+
+        self._version = completed.stdout.strip()
+        self._name_format = '{name}/{version}-{hash}'
+
+    def name(self):
+        return 'spack'
+
+    def version(self):
+        return self._version
+
+    def modulecmd(self, *args):
+        return ' '.join(['spack', *args])
+
+    def _execute(self, cmd, *args):
+        modulecmd = self.modulecmd(cmd, *args)
+        completed = osext.run_command(modulecmd, check=True)
+        return completed.stdout
+
+    def available_modules(self, substr):
+        output = self.execute('find', '--format', self._name_format,
+                              substr)
+        ret = []
+        for line in output.split('\n'):
+            if not line or line[-1] == ':':
+                # Ignore empty lines and path entries
+                continue
+
+            ret.append(Module(line))
+
+        return ret
+
+    def loaded_modules(self):
+        output = self.execute('find', '--loaded', '--format',
+                              self._name_format)
+        return [Module(m) for m in output.split('\n') if m]
+
+    def conflicted_modules(self, module):
+        return []
+
+    def is_module_loaded(self, module):
+        module = self.execute('find', '--format', self._name_format, name)
+        module = Module(module)
+        return module in self.loaded_modules()
+
+    def load_module(self, module):
+        pass
+
+    def unload_module(self, module):
+        pass
+
+    def unload_all(self):
+        pass
+
+    def searchpath(self):
+        return []
+
+    def searchpath_add(self, *dirs):
+        pass
+
+    def searchpath_remove(self, *dirs):
+        pass
+
+    def emit_load_instr(self, module):
+        return f'spack load {module}'
+
+    def emit_unload_instr(self, module):
+        return f'spack unload {module}'

reframe/schemas/config.json

@@ -156,8 +156,8 @@
                     },
                     "modules_system": {
                         "type": "string",
-                        "enum": ["tmod", "tmod31", "tmod32",
-                                 "tmod4", "lmod", "nomod"]
+                        "enum": ["tmod", "tmod31", "tmod32", "tmod4",
+                                 "lmod", "nomod", "spack"]
                     },
                     "modules": {"$ref": "#/defs/modules_list"},
                     "variables": {"$ref": "#/defs/envvar_list"},

unittests/fixtures.py

@@ -10,9 +10,7 @@
 import inspect
 import os
 import sys
-import tempfile
 
-import reframe
 import reframe.core.config as config
 import reframe.core.modules as modules
 import reframe.core.runtime as rt
@@ -80,7 +78,7 @@ def environment_by_name(name, partition):
 
 def has_sane_modules_system():
     return not isinstance(rt.runtime().modules_system.backend,
-                          modules.NoModImpl)
+                          (modules.NoModImpl, modules.SpackImpl))
 
 
 def custom_prefix(prefix):

unittests/test_cli.py

@@ -642,7 +642,7 @@ def test_unload_module(run_reframe, user_exec_ctx):
     # more exhaustively.
 
     ms = rt.runtime().modules_system
-    if ms.name == 'nomod':
+    if not fixtures.has_sane_modules_system():
         pytest.skip('no modules system found')
 
     with rt.module_use('unittests/modules'):
@@ -661,7 +661,7 @@ def test_unload_module(run_reframe, user_exec_ctx):
 
 def test_unuse_module_path(run_reframe, user_exec_ctx):
     ms = rt.runtime().modules_system
-    if ms.name == 'nomod':
+    if not fixtures.has_sane_modules_system():
         pytest.skip('no modules system found')
 
     module_path = 'unittests/modules'
@@ -679,7 +679,7 @@ def test_unuse_module_path(run_reframe, user_exec_ctx):
 
 def test_use_module_path(run_reframe, user_exec_ctx):
     ms = rt.runtime().modules_system
-    if ms.name == 'nomod':
+    if not fixtures.has_sane_modules_system():
         pytest.skip('no modules system found')
 
     module_path = 'unittests/modules'
@@ -696,7 +696,7 @@ def test_use_module_path(run_reframe, user_exec_ctx):
 
 def test_overwrite_module_path(run_reframe, user_exec_ctx):
     ms = rt.runtime().modules_system
-    if ms.name == 'nomod':
+    if not fixtures.has_sane_modules_system():
         pytest.skip('no modules system found')
 
     module_path = 'unittests/modules'

unittests/test_modules.py

@@ -12,7 +12,7 @@
 from reframe.core.exceptions import ConfigError, EnvironError
 
 
-@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'nomod'])
+@pytest.fixture(params=['tmod', 'tmod4', 'lmod', 'spack', 'nomod'])
 def modules_system(request, monkeypatch):
     # Always pretend to be on a clean modules environment
     monkeypatch.setenv('MODULEPATH', '')
@@ -31,7 +31,7 @@ def modules_system(request, monkeypatch):
 
 
 def test_searchpath(modules_system):
-    if modules_system.name == 'nomod':
+    if modules_system.name in ['nomod', 'spack']:
         # Simply test that no exceptions are thrown
         modules_system.searchpath_remove(fixtures.TEST_MODULES)
     else:
@@ -70,7 +70,7 @@ def module_collection(modules_system, tmp_path, monkeypatch):
 
 
 def test_module_load(modules_system):
-    if modules_system.name == 'nomod':
+    if modules_system.name in ['nomod', 'spack']:
         modules_system.load_module('foo')
         modules_system.unload_module('foo')
     else:
@@ -100,7 +100,7 @@ def test_module_load_collection(modules_system, module_collection):
 
 
 def test_module_load_force(modules_system):
-    if modules_system.name == 'nomod':
+    if modules_system.name in ['nomod', 'spack']:
         modules_system.load_module('foo', force=True)
     else:
         modules_system.load_module('testmod_foo')
@@ -150,14 +150,19 @@ def test_module_unload_all(modules_system):
 def test_module_list(modules_system):
     if modules_system.name == 'nomod':
         assert 0 == len(modules_system.loaded_modules())
+    elif modules_system.name == 'spack':
+        # If Spack is installed, we can't be sure that the user has not loaded
+        # any module and we cannot unload them in here, since we don't have
+        # Python bindings. So we only check that we get a list back.
+        assert isinstance(modules_system.loaded_modules(), list)
     else:
         modules_system.load_module('testmod_foo')
         assert 'testmod_foo' in modules_system.loaded_modules()
         modules_system.unload_module('testmod_foo')
 
 
 def test_module_conflict_list(modules_system):
-    if modules_system.name == 'nomod':
+    if modules_system.name in ['nomod', 'spack']:
         assert 0 == len(modules_system.conflicted_modules('foo'))
     else:
         conflict_list = modules_system.conflicted_modules('testmod_bar')
@@ -169,12 +174,17 @@ def test_module_available_all(modules_system):
     modules = sorted(modules_system.available_modules())
     if modules_system.name == 'nomod':
         assert modules == []
+    elif modules_system.name == 'spack':
+        # If Spack is installed, we can't fool it with environment variables
+        # about its installed packages, like we can do with modules, so we
+        # simply check that we get a list back.
+        assert isinstance(modules, list)
     else:
         assert (modules == ['testmod_bar', 'testmod_base',
                             'testmod_boo', 'testmod_ext', 'testmod_foo'])
 
 
-def test_module_available_substr(modules_system):
+def _test_module_available_substr(modules_system):
     modules = sorted(modules_system.available_modules('testmod_b'))
     if modules_system.name == 'nomod':
         assert modules == []
@@ -214,6 +224,13 @@ def _emit_load_commands_lmod(modules_system):
     assert [emit_cmds('m0')] == ['module load m1', 'module load m2']
 
 
+def _emit_load_commands_spack(modules_system):
+    emit_cmds = modules_system.emit_load_commands
+    assert [emit_cmds('foo')] == ['spack load foo']
+    assert [emit_cmds('foo/1.2')] == ['spack load foo/1.2']
+    assert [emit_cmds('m0')] == ['spack load m1', 'spack load m2']
+
+
 def _emit_load_commands_nomod(modules_system):
     emit_cmds = modules_system.emit_load_commands
     assert [emit_cmds('foo')] == []
@@ -252,6 +269,13 @@ def _emit_unload_commands_lmod(modules_system):
     assert [emit_cmds('m0')] == ['module unload m2', 'module unload m1']
 
 
+def _emit_unload_commands_spack(modules_system):
+    emit_cmds = modules_system.emit_unload_commands
+    assert [emit_cmds('foo')] == ['spack unload foo']
+    assert [emit_cmds('foo/1.2')] == ['spack unload foo/1.2']
+    assert [emit_cmds('m0')] == ['spack unload m2', 'spack unload m1']
+
+
 def _emit_unload_commands_nomod(modules_system):
     emit_cmds = modules_system.emit_unload_commands
     assert [emit_cmds('foo')] == []