Merge pull request #1719 from rsarm/feat/easybuild

[feat] Add support for building tests using EasyBuild

Author: Vasileios Karakasis

docs/config_reference.rst

@@ -1276,6 +1276,18 @@ It can either be a simple string or a JSON object with the following attributes:
    A boolean value indicating whether this module refers to a module collection.
    Module collections are treated differently from simple modules when loading.
 
+.. js:attribute:: .path
+
+   :required: No
+   :default: ``null``
+
+   If the module is not present in the default ``MODULEPATH``, the module's location can be specified here.
+   ReFrame will make sure to set and restore the ``MODULEPATH`` accordingly for loading the module.
+
+
+   .. versionadded:: 3.5.0
+
+
 .. seealso::
 
    Module collections with `Environment Modules <https://modules.readthedocs.io/en/latest/MIGRATING.html#module-collection>`__ and `Lmod <https://lmod.readthedocs.io/en/latest/010_user.html#user-collections>`__.

reframe/core/buildsystems.py

@@ -5,6 +5,7 @@
 
 import abc
 import os
+import re
 
 import reframe.core.fields as fields
 import reframe.utility.typecheck as typ
@@ -122,8 +123,8 @@ def __init__(self):
     def emit_build_commands(self, environ):
         '''Return the list of commands for building using this build system.
 
-        The build commands may always assume to be issued from the top-level
-        directory of the code that is to be built.
+        The build commands, as well as this function, will always be executed
+        from the test's stage directory.
 
         :arg environ: The programming environment for which to emit the build
            instructions.
@@ -132,8 +133,24 @@ def emit_build_commands(self, environ):
         :raises: :class:`BuildSystemError` in case of errors when generating
           the build instructions.
 
-        .. note::
-            This method is relevant only to developers of new build systems.
+        .. versionchanged:: 3.5.0
+           This function executes from the test stage directory.
+
+        :meta private:
+
+        '''
+
+    def post_build(self, buildjob):
+        '''Callback function that the framework will call when the compilation
+        is done.
+
+        Build systems may use this information to do some post processing and
+        provide additional, build system-specific, functionality to the users.
+
+        .. versionadded:: 3.5.0
+
+        :meta private:
+
         '''
 
     def _resolve_flags(self, flags, environ):
@@ -655,6 +672,113 @@ def emit_build_commands(self, environ):
         return prepare_cmd + [' '.join(configure_cmd), ' '.join(make_cmd)]
 
 
+class EasyBuild(BuildSystem):
+    '''A build system for building test code using `EasyBuild
+    <https://easybuild.io/>`__.
+
+    ReFrame will use EasyBuild to build and install the code in the test's
+    stage directory by default. ReFrame uses environment variables to
+    configure EasyBuild for running, so Users can pass additional options to
+    the ``eb`` command and modify the default behaviour.
+
+    .. versionadded:: 3.5.0
+
+    '''
+
+    #: The list of easyconfig files to build and install.
+    #: This field is required.
+    #:
+    #: :type: :class:`List[str]`
+    #: :default: ``[]``
+    easyconfigs = fields.TypedField(typ.List[str])
+
+    #: Options to pass to the ``eb`` command.
+    #:
+    #: :type: :class:`List[str]`
+    #: :default: ``[]``
+    options = fields.TypedField(typ.List[str])
+
+    #: Instruct EasyBuild to emit a package for the built software.
+    #: This will essentially pass the ``--package`` option to ``eb``.
+    #:
+    #: :type: :class:`bool`
+    #: :default: ``[]``
+    emit_package = fields.TypedField(bool)
+
+    #: Options controlling the package creation from EasyBuild.
+    #: For each key/value pair of this dictionary, ReFrame will pass
+    #: ``--package-{key}={val}`` to the EasyBuild invocation.
+    #:
+    #: :type: :class:`Dict[str, str]`
+    #: :default: ``{}``
+    package_opts = fields.TypedField(typ.Dict[str, str])
+
+    #: Default prefix for the EasyBuild installation.
+    #:
+    #: Relative paths will be appended to the stage directory of the test.
+    #: ReFrame will set the following environment variables before running
+    #: EasyBuild.
+    #:
+    #: .. code-block:: bash
+    #:
+    #:    export EASYBUILD_BUILDPATH={prefix}/build
+    #:    export EASYBUILD_INSTALLPATH={prefix}
+    #:    export EASYBUILD_PREFIX={prefix}
+    #:    export EASYBUILD_SOURCEPATH={prefix}
+    #:
+    #: Users can change these defaults by passing specific options to the
+    #: ``eb`` command.
+    #:
+    #: :type: :class:`str`
+    #: :default: ``easybuild``
+    prefix = fields.TypedField(str)
+
+    def __init__(self):
+        super().__init__()
+        self.easyconfigs = []
+        self.options = []
+        self.emit_package = False
+        self.package_opts = {}
+        self.prefix = 'easybuild'
+        self._eb_modules = []
+        self._prefix_save = None
+
+    def emit_build_commands(self, environ):
+        if not self.easyconfigs:
+            raise BuildSystemError(f"'easyconfigs' must not be empty")
+
+        easyconfigs = ' '.join(self.easyconfigs)
+        if self.emit_package:
+            self.options.append('--package')
+            for key, val in self.package_opts.items():
+                self.options.append(f'--package-{key}={val}')
+
+        prefix = os.path.join(os.getcwd(), self.prefix)
+        options = ' '.join(self.options)
+        self._prefix_save = prefix
+        return [f'export EASYBUILD_BUILDPATH={prefix}/build',
+                f'export EASYBUILD_INSTALLPATH={prefix}',
+                f'export EASYBUILD_PREFIX={prefix}',
+                f'export EASYBUILD_SOURCEPATH={prefix}',
+                f'eb {easyconfigs} {options}']
+
+    def post_build(self, buildjob):
+        # Store the modules generated by EasyBuild
+
+        modulesdir = os.path.join(self._prefix_save, 'modules', 'all')
+        with open(buildjob.stdout) as fp:
+            out = fp.read()
+
+        self._eb_modules = [
+            {'name': m, 'collection': False, 'path': modulesdir}
+            for m in re.findall(r'building and installing (\S+)...', out)
+        ]
+
+    @property
+    def generated_modules(self):
+        return self._eb_modules
+
+
 class BuildSystemField(fields.TypedField):
     def __init__(self, fieldname, *other_types):
         super().__init__(fieldname, BuildSystem, *other_types)

reframe/core/environments.py

@@ -20,7 +20,7 @@ def normalize_module_list(modules):
     ret = []
     for m in modules:
         if isinstance(m, str):
-            ret.append({'name': m, 'collection': False})
+            ret.append({'name': m, 'collection': False, 'path': None})
         else:
             ret.append(m)