Merge branch 'master' into enhancement/assert_reference_restriction

Author: vkarak

Jenkinsfile

@@ -83,29 +83,14 @@ stage('Unittest') {
                                             echo \$SCRATCH""").trim()
                 def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}"
                 def moduleDefinition = ''
-                def gitClone = ''
                 if (machineName == 'leone') {
                     moduleDefinition = '''module() { eval `/usr/bin/modulecmd bash $*`; }
                                           export -f module'''
-                    gitClone = """module use /apps/common/UES/RHAT6/easybuild/modules/all/
-                                  module load git
-                                  git init .
-                                  git fetch --tags --quiet https://github.com/eth-cscs/reframe.git +refs/heads/*:refs/remotes/origin/*
-                                  git config remote.origin.url https://github.com/eth-cscs/reframe.git
-                                  git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
-                                  git config remote.origin.url https://github.com/eth-cscs/reframe.git
-                                  git fetch --tags --quiet https://github.com/eth-cscs/reframe.git +refs/pull/*:refs/remotes/origin/pr/* +refs/heads/master:refs/remotes/origin/master
-                                  git rev-parse ${env.ghprbActualCommit}^{commit}
-                                  git config core.sparsecheckout
-                                  git checkout -f ${env.ghprbActualCommit}"""
                 }
                 dir(reframeDir) {
-                    if (machineName != 'leone') {
-                        checkout scm
-                    }
+                    checkout scm
                     sh("""${loginBash}
                           ${moduleDefinition}
-                          ${gitClone}
                           ln -sf ../${cscsSettings} reframe/settings.py
                           bash ${reframeDir}/${bashScript} -f ${reframeDir} -i ''""")
                 }

cscs-checks/apps/espresso/espresso_check.py

@@ -15,9 +15,11 @@ def __init__(self, variant, **kwargs):
                                        'Espresso')
         self.executable = 'pw.x'
         self.executable_opts = '-in ausurf.in'.split()
-
-        self.sanity_patterns = sn.assert_found(
-            r'convergence has been achieved', self.stdout)
+        energy = sn.extractsingle(r'!\s+total energy\s+=\s+(?P<energy>\S+) Ry',
+                                  self.stdout, 'energy', float)
+        self.sanity_patterns = sn.all([
+            sn.assert_found(r'convergence has been achieved', self.stdout),
+            sn.assert_reference(energy, -11427.08612278, -1e-10, 1e-10)])
         self.perf_patterns = {
             'sec': sn.extractsingle(r'electrons    :\s+(?P<sec>\S+)s CPU ',
                                     self.stdout, 'sec', float)

docs/running.rst

@@ -412,30 +412,37 @@ By default, the output in ``reframe.log`` looks like the following:
 
 .. code-block:: none
 
-  [2017-10-24T21:19:04] info: reframe: [----------] started processing example7_check (Matrix-vector mult
-  iplication example with Cuda)
-  [2017-10-24T21:19:04] info: reframe: [   SKIP   ] skipping daint:mc
-  [2017-10-24T21:19:04] info: reframe: [ RUN      ] example7_check on daint:gpu using PrgEnv-cray
-  [2017-10-24T21:19:04] debug: example7_check: setting up the environment
-  [2017-10-24T21:19:04] debug: example7_check: loading environment for partition daint:gpu
-  [2017-10-24T21:19:05] debug: example7_check: loading environment PrgEnv-cray
-  [2017-10-24T21:19:05] debug: example7_check: setting up paths
-  [2017-10-24T21:19:05] debug: example7_check: setting up the job descriptor
-  [2017-10-24T21:19:05] debug: example7_check: job scheduler backend: nativeslurm
-  [2017-10-24T21:19:05] debug: example7_check: setting up performance logging
-  [2017-10-24T21:19:05] debug: example7_check: compilation started
-  [2017-10-24T21:19:06] debug: example7_check: compilation stdout:
-
-  [2017-10-24T21:19:06] debug: example7_check: compilation stderr:
-  nvcc warning : The 'compute_20', 'sm_20', and 'sm_21' architectures are deprecated, and may be removed
-  in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
-
-  [2017-10-24T21:19:06] debug: example7_check: compilation finished
-  [2017-10-24T21:19:09] debug: example7_check: spawned job (jobid=4163846)
-  [2017-10-24T21:19:21] debug: example7_check: spawned job finished
-  [2017-10-24T21:19:21] debug: example7_check: copying interesting files to output directory
-  [2017-10-24T21:19:21] debug: example7_check: removing stage directory
-  [2017-10-24T21:19:21] info: reframe: [       OK ] example7_check on daint:gpu using PrgEnv-cray
+   [2018-03-01T20:03:13] info: reframe: [ RUN      ] example7_check on daint:gpu using PrgEnv-cray
+   [2018-03-01T20:03:13] debug: example7_check: entering stage: setup
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: loading environment for the current partition
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: executing OS command: modulecmd python show daint-gpu
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: executing OS command: modulecmd python load daint-gpu
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: loading test's environment
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: executing OS command: modulecmd python show cudatoolkit
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: executing OS command: modulecmd python load cudatoolkit
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: setting up paths
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: setting up the job descriptor
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: job scheduler backend: local
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: setting up performance logging
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: entering stage: compile
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: copying /users/karakasv/Devel/reframe/tutorial/src to stage directory (/users/karakasv/Devel/reframe/stage/gpu/example7_check/PrgEnv
+   -cray)
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: symlinking files: []
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: Staged sourcepath: /users/karakasv/Devel/reframe/stage/gpu/example7_check/PrgEnv-cray/example_matrix_vector_multiplication_cuda.cu
+   [2018-03-01T20:03:13] debug: example7_check on daint:gpu using PrgEnv-cray: executing OS command: nvcc  -O3 -I/users/karakasv/Devel/reframe/stage/gpu/example7_check/PrgEnv-cray /users/karakasv/Devel/reframe/s
+   tage/gpu/example7_check/PrgEnv-cray/example_matrix_vector_multiplication_cuda.cu -o /users/karakasv/Devel/reframe/stage/gpu/example7_check/PrgEnv-cray/./example7_check
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: compilation stdout:
+
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: compilation stderr:
+   nvcc warning : The 'compute_20', 'sm_20', and 'sm_21' architectures are deprecated, and may be removed in a future release (Use -Wno-deprecated-gpu-targets to suppress warning).
+
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: compilation finished
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: entering stage: run
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: executing OS command: sbatch /users/karakasv/Devel/reframe/stage/gpu/example7_check/PrgEnv-cray/example7_check_daint_gpu_PrgEnv-cray
+   .sh
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: spawned job (jobid=6224537)
+   [2018-03-01T20:03:14] debug: example7_check on daint:gpu using PrgEnv-cray: entering stage: wait
+
 
 Each line starts with a timestamp, the level of the message (``info``, ``debug`` etc.), the context in which the framework is currently executing (either ``reframe`` or the name of the current test and, finally, the actual message.
 
@@ -459,7 +466,7 @@ The default configuration looks as follows:
           'reframe.log' : {
               'level'     : 'DEBUG',
               'format'    : '[%(asctime)s] %(levelname)s: '
-                            '%(testcase_name)s: %(message)s',
+                            '%(check_info)s: %(message)s',
               'append'    : False,
           },
 
@@ -500,9 +507,9 @@ The configurable properties of a log record handler are the following:
     If ReFrame is not in the context of regression test, ``reframe`` will be printed.
   * ``check_jobid``: Prints the job or process id of the job or process associated with currently executing regression test.
     If a job or process is not yet created, ``-1`` will be printed.
-  * ``testcase_name``: Print the name of the test case that is currently executing.
-    Test case is essentially a tuple consisting of the test name, the current system and partition and the current programming environment.
-    This format string prints out like ``<test-name>@<partition> using <environ>``.
+  * ``check_info``: Print live information of the currently executing check.
+    By default this field has the form ``<check_name> on <current_partition> using <current_environment>``.
+    It can be configured on a per test basis by overriding the :func:`info <reframe.core.pipeline.RegressionTest.info>` method in your regression test.
 
 * ``datefmt`` (default: ``'%FT%T'``) The format that will be used for outputting timestamps (i.e., the ``%(asctime)s`` field).
   Acceptable formats must conform to standard library's `time.strftime() <https://docs.python.org/3.6/library/time.html#time.strftime>`__ function.
@@ -513,21 +520,26 @@ The configurable properties of a log record handler are the following:
   If set for a ``filename.log`` handler entry, the resulting log file name will be ``filename_<timestamp>.log``.
   This property is ignored for the standard output/error handlers.
 
+.. note::
+   .. versionchanged:: 2.10
+      The ``testcase_name`` logging attribute was replaced with the ``check_info``, which is now also configurable
+
+
 Performance Logging
 ^^^^^^^^^^^^^^^^^^^
 
 ReFrame supports additional logging for performance tests specifically, in order to record historical performance data.
 For each performance test, a log file of the form ``<test-name>.log`` is created under the ReFrame's `log directory <#configuring-reframe-directories>`__ where the test's performance is recorded.
-The default format used for this file is ``'[%(asctime)s] %(testcase_name)s (jobid=%(check_jobid)s): %(message)s'`` and ReFrame always appends to this file.
+The default format used for this file is ``'[%(asctime)s] %(check_info)s (jobid=%(check_jobid)s): %(message)s'`` and ReFrame always appends to this file.
 Currently, it is not possible for users to configure performance logging.
 
 The resulting log file looks like the following:
 
 .. code-block:: none
 
-  [2017-12-01T15:31:20] example7_check@daint:gpu using PrgEnv-cray (jobid=649790): value: 47.797996, reference: (50.0, -0.1, 0.1)
-  [2017-12-01T15:31:24] example7_check@daint:gpu using PrgEnv-gnu (jobid=649791): value: 49.048228, reference: (50.0, -0.1, 0.1)
-  [2017-12-01T15:31:24] example7_check@daint:gpu using PrgEnv-pgi (jobid=649792): value: 48.575334, reference: (50.0, -0.1, 0.1)
+   [2018-03-01T20:01:20] reframe 2.10: example7_check on daint:gpu using PrgEnv-pgi (jobid=6224525): value: 49.637615, reference: (50.0, -0.1, 0.1)
+   [2018-03-01T20:01:20] reframe 2.10: example7_check on daint:gpu using PrgEnv-cray (jobid=6224523): value: 49.931819, reference: (50.0, -0.1, 0.1)
+   [2018-03-01T20:01:21] reframe 2.10: example7_check on daint:gpu using PrgEnv-gnu (jobid=6224524): value: 49.428855, reference: (50.0, -0.1, 0.1)
 
 The interpretation of the performance values depends on the individual tests.
 The above output is from the CUDA performance test we presented in the `tutorial <tutorial.html#writing-a-performance-test>`__, so the value refers to the achieved Gflop/s.
@@ -617,3 +629,78 @@ Here is an example output of ReFrame using asynchronous execution policy:
 
 The asynchronous execution policy may provide significant overall performance benefits for run-only regression tests.
 For compile-only and normal tests that require a compilation, the execution time will be bound by the total compilation time of the test.
+
+
+Manipulating modules
+--------------------
+
+.. versionadded:: 2.11
+
+ReFrame allows you to change the modules loaded by a regression test on-the-fly without having to edit the regression test file.
+This feature is extremely useful when you need to quickly test a newer version of a module, but it also allows you to completely decouple the module names used in your regression tests from the real module names in a system, thus making your test even more portable.
+This is achieved by defining *module mappings*.
+
+There are two ways to pass module mappings to ReFrame.
+The first is to use the ``--map-module`` command-line option, which accepts a module mapping.
+For example, the following line maps the module ``test_module`` to the module ``real_module``:
+
+.. code-block:: none
+
+  --map-module='test_module: real_module'
+
+In this case, whenever ReFrame is asked to load ``test_module``, it will load ``real_module``.
+Any string without spaces may be accepted in place of ``test_module`` and ``real_module``.
+You can also define multiple module mappings at once by repeating the ``--map-module``.
+If more than one mapping is specified for the same module, then the last mapping will take precedence.
+It is also possible to map a single module to more than one target.
+This can be done by listing the target modules separated by spaces in the order that they should be loaded.
+In the following example, ReFrame will load ``real_module0`` and ``real_module1`` whenever the ``test_module`` is encountered:
+
+.. code-block:: none
+
+  --map-module 'test_module: real_module0 real_module1'
+
+The second way of defining mappings is by listing them on a file, which you can then pass to ReFrame through the command-line option ``--module-mappings``.
+Each line on the file corresponds to the definition of a mapping for a single module.
+The syntax of the individual mappings in the file is the same as with the option ``--map-module`` and the same rules apply regarding repeated definitions.
+Text starting with ``#`` is considered a comment and is ignored until the end of line is encountered.
+Empty lines are ignored.
+The following block shows an example of module mapping file:
+
+.. code-block:: none
+
+  module-1: module-1a  # an inline comment
+  module-2: module-2a module-2b module-2c
+
+  # This is a full line comment
+  module-4: module-4a module-4b
+
+If both ``--map-module`` and ``--module-mappings`` are passed, ReFrame will first create a mapping from the definitions on the file and it will then process the definitions passed with the ``--map-module`` options.
+As usual, later definitions will override the former.
+
+A final note on module mappings.
+Module mappings can be arbitrarily deep as long as they do not form a cycle.
+In this case, ReFrame will issue an error (denoting the offending cyclic dependency).
+For example, suppose having the following mapping file:
+
+.. code-block:: none
+
+   cudatoolkit: foo
+   foo: bar
+   bar: foobar
+   foobar: cudatoolkit
+
+If you now try to run a test that loads the module `cudatoolkit`, the following error will be yielded:
+
+.. code-block:: none
+
+   ------------------------------------------------------------------------------
+   FAILURE INFO for example7_check
+     * System partition: daint:gpu
+     * Environment: PrgEnv-gnu
+     * Stage directory: None
+     * Job type: batch job (id=-1)
+     * Maintainers: ['you-can-type-your-email-here']
+     * Failing phase: setup
+     * Reason: caught framework exception: module cyclic dependency: cudatoolkit->foo->bar->foobar->cudatoolkit
+   ------------------------------------------------------------------------------

reframe/__init__.py

@@ -1,11 +1,7 @@
-#
-# Sanity checks and modules environment setup
-#
-
 import sys
 
 
-VERSION = '2.10'
+VERSION = '2.11'
 _required_pyver = (3, 5, 0)
 
 # Check python version

reframe/core/modules.py

@@ -5,6 +5,7 @@
 import abc
 import os
 import re
+from collections import OrderedDict
 
 import reframe.core.fields as fields
 import reframe.utility.os as os_ext
@@ -203,6 +204,35 @@ def is_module_loaded(self, name):
     def _is_module_loaded(self, name):
         return self._backend.is_module_loaded(Module(name))
 
+    def load_mapping(self, mapping):
+        """Updates the internal module mapping with a single mapping"""
+        key, *rest = mapping.split(':')
+        if len(rest) != 1:
+            raise ConfigError('invalid mapping syntax: %s' % mapping)
+
+        key = key.strip()
+        values = rest[0].split()
+        if not key:
+            raise ConfigError('no key found in mapping: %s' % mapping)
+
+        if not values:
+            raise ConfigError('no mapping defined for module: %s' % key)
+
+        self.module_map[key] = list(OrderedDict.fromkeys(values))
+
+    def load_mapping_from_file(self, filename):
+        """Update the internal module mapping from mappings in a file."""
+        with open(filename) as fp:
+            for lineno, line in enumerate(fp, start=1):
+                line = line.strip().split('#')[0]
+                if not line:
+                    continue
+
+                try:
+                    self.load_mapping(line)
+                except ConfigError as e:
+                    raise ConfigError('%s:%s' % (filename, lineno)) from e
+
     @property
     def name(self):
         """Return the name of this module system."""
@@ -513,7 +543,7 @@ def init_modules_system(modules_kind=None):
     elif modules_kind == 'lmod':
         _modules_system = ModulesSystem(LModImpl())
     else:
-        raise ConfigError('unknown module system')
+        raise ConfigError('unknown module system: %s' % modules_kind)
 
 
 def get_modules_system():

reframe/core/pipeline.py

@@ -636,10 +636,21 @@ def info(self):
 
         This method is used by the front-end to print the status message during
         the test's execution.
+        This function is also called to provide the message for the
+        ``check_info`` `logging attribute <running.html#logging>`__.
+        By default, it returns a message reporting the test name, the current
+        partition and the current programming environment that the test is
+        currently executing on.
+
+        :returns: a string with an informational message about this test
+
+        .. note ::
+           When overriding this method, you should pay extra attention on how
+           you use the :class:`RegressionTest`'s attributes, because this
+           method may be called at any point of the test's lifetime.
+
+           .. versionadded:: 2.10
 
-        :returns: a string with an informational message containing the test
-            name, the current partition and the current programming environment
-            that the test is currently executing on.
         """
         ret = self.name
         if self.current_partition: