Merge branch 'master' into docs/testdeps

Vasileios Karakasis · web-flow · commit 91e9cbf315c0 · 2019-12-17T15:16:26.000+01:00
diff --git a/README.md b/README.md
@@ -95,3 +95,9 @@ For unsubscribing, you may send an empty message to [reframe-unsubscribe@sympa.c
 ### Slack
 
 You may also reach the community through Slack [here](https://reframe-slack.herokuapp.com).
+
+
+## Contributing back
+
+ReFrame is an open-source project and we welcome third-party contributions.
+Check out our Contribution Guide [here](https://github.com/eth-cscs/reframe/wiki/contributing-to-reframe).
diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash
@@ -50,7 +50,7 @@ run_tutorial_checks()
 run_user_checks()
 {
     cmd="./bin/reframe -C config/cscs.py --exec-policy=async --save-log-files \
--r --flex-alloc-tasks=2 -t production|benchmark $@"
+-r --flex-alloc-nodes=2 -t production|benchmark $@"
     echo "Running user checks with \`$cmd'"
     checked_exec $cmd
 }
diff --git a/config/cscs.py b/config/cscs.py
@@ -189,9 +189,6 @@ class ReframeSettings:
                                      'PrgEnv-pgi'],
                         'descr': 'Hybrid nodes (Haswell/P100)',
                         'max_jobs': 100,
-                        'resources': {
-                            'switches': ['--switches={num_switches}']
-                        }
                     },
 
                     'mc': {
diff --git a/cscs-checks/libraries/io/netcdf_compile_run.py b/cscs-checks/libraries/io/netcdf_compile_run.py
@@ -63,7 +63,24 @@ def setup(self, partition, environ, **job_opts):
                     '-lnetcdf', '-lnetcdf_c++4', '-lnetcdff'
                 ]
             elif environ.name == 'PrgEnv-pgi-nompi':
-                self.build_system.ldflags = ['-B' + self.linkage]
+                self.modules = ['netcdf/4.6.1-pgi-18.5-gcc-5.4.0-2.26',
+                                'netcdf-c++/4.3.0-pgi-18.5-gcc-5.4.0-2.26',
+                                'netcdf-fortran/4.4.4-pgi-18.5-gcc-5.4.0-2.26']
+                self.build_system.ldflags = [
+                    '-B' + self.linkage,
+                    '-L$EBROOTNETCDF/lib',
+                    '-L$EBROOTNETCDFMINCPLUSPLUS/lib',
+                    '-L$EBROOTNETCDFMINFORTRAN/lib',
+                    '-L$EBROOTNETCDF/lib64',
+                    '-L$EBROOTNETCDFMINCPLUSPLUS/lib64',
+                    '-L$EBROOTNETCDFMINFORTRAN/lib64',
+                    '-lnetcdf', '-lnetcdf_c++4', '-lnetcdff'
+                ]
+                self.build_system.fflags = [
+                    '-I$EBROOTNETCDF/include',
+                    '-I$EBROOTNETCDFMINCPLUSPLUS/include',
+                    '-I$EBROOTNETCDFMINFORTRAN/include'
+                ]
         else:
             self.build_system.ldflags = ['-%s' % self.linkage]
 
diff --git a/cscs-checks/mch/collectives_halo.py b/cscs-checks/mch/collectives_halo.py
@@ -108,9 +108,9 @@ def __init__(self, variant):
         super().__init__(variant,
                          {
                              'kesch': {
-                                 'nocomm':  5.7878,
-                                 'nocomp':  5.62155,
-                                 'default': 5.53777
+                                 'nocomm':  6.89819,
+                                 'nocomp':  6.98276,
+                                 'default': 6.85289
                              },
                              'daint': {
                                  'nocomm':  0.0171947,
@@ -130,8 +130,8 @@ def __init__(self, variant):
                          {
                              'kesch': {
                                  'nocomm':  5.7878,
-                                 'nocomp':  5.62155,
-                                 'default': 5.53777
+                                 'nocomp':  54.2012,
+                                 'default': 55.142
                              },
                              'daint': {
                                  'nocomm':  0.978306,
diff --git a/cscs-checks/microbenchmarks/alloc_speed/alloc_speed.py b/cscs-checks/microbenchmarks/alloc_speed/alloc_speed.py
@@ -47,10 +47,10 @@ def __init__(self, hugepages):
                     'time': (1.41, None, 0.05, 's')
                 },
                 'kesch:cn': {
-                    'time': (1.25, None, 0.10, 's')
+                    'time': (1.41, None, 0.10, 's')
                 },
                 'kesch:pn': {
-                    'time': (0.55, None, 0.10, 's')
+                    'time': (0.70, None, 0.10, 's')
                 },
                 '*': {
                     'time': (0, None, None, 's')
diff --git a/cscs-checks/prgenv/helloworld.py b/cscs-checks/prgenv/helloworld.py
@@ -137,6 +137,9 @@ def __init__(self, lang, linkage):
             'PrgEnv-intel': ['-qopenmp'],
             'PrgEnv-pgi': ['-mp']
         }
+        if self.current_system.name == 'kesch':
+            self.prgenv_flags['PrgEnv-cray'] = ['-homp']
+
         self.num_tasks = 1
         self.num_tasks_per_node = 1
         self.num_cpus_per_task = 4
@@ -193,6 +196,9 @@ def __init__(self, lang, linkage):
             'PrgEnv-intel': ['-qopenmp'],
             'PrgEnv-pgi': ['-mp']
         }
+        if self.current_system.name == 'kesch':
+            self.prgenv_flags['PrgEnv-cray'] = ['-homp']
+
         self.num_tasks = 6
         self.num_tasks_per_node = 3
         self.num_cpus_per_task = 4
diff --git a/cscs-checks/tools/profiling_and_debugging/perftools.py b/cscs-checks/tools/profiling_and_debugging/perftools.py
@@ -75,7 +75,7 @@ def __init__(self, lang):
         self.post_run = ['grep INFO rfm_*_build.err']
         self.sanity_patterns = sn.all([
             sn.assert_found('SUCCESS', self.stdout),
-            sn.assert_found(r'^INFO: creating the CrayPat-instrumented exec',
+            sn.assert_found(r'^INFO: creating the (\w+)-instrumented exec',
                             self.stdout),
             sn.assert_found('Table 1:  Profile by Function', self.stdout),
         ])
diff --git a/docs/configure.rst b/docs/configure.rst
@@ -103,7 +103,7 @@ Each system is a key/value pair, with the key being the name of the system and t
 The valid attributes of a system are the following:
 
 * ``descr``: A detailed description of the system (default is the system name).
-* ``hostnames``: This is a list of hostname patterns that will be used by ReFrame when it tries to `auto-detect <#system-auto-detection>`__ the current system (default ``[]``).
+* ``hostnames``: This is a list of hostname patterns according to the `Python Regular Expression Syntax <https://docs.python.org/3/library/re.html#regular-expression-syntax>`__ , which will be used by ReFrame when it tries to `auto-detect <#system-auto-detection>`__ the current system (default ``[]``).
 * ``modules_system``: *[new in 2.8]* The modules system that should be used for loading environment modules on this system (default :class:`None`).
   Three types of modules systems are currently supported:
 
@@ -382,7 +382,7 @@ System Auto-Detection
 When ReFrame is launched, it tries to detect the current system and select the correct site configuration entry. The auto-detection process is as follows:
 
 ReFrame first tries to obtain the hostname from ``/etc/xthostname``, which provides the unqualified *machine name* in Cray systems.
-If this cannot be found the hostname will be obtained from the standard ``hostname`` command. 
+If this cannot be found the hostname will be obtained from the standard ``hostname`` command.
 Having retrieved the hostname, ReFrame goes through all the systems in its configuration and tries to match the hostname against any of the patterns in the ``hostnames`` attribute of `system configuration <#system-configuration>`__.
 The detection process stops at the first match found, and the system it belongs to is considered as the current system.
 If the system cannot be auto-detected, ReFrame will issue a warning and fall back to a generic system configuration, which is equivalent to the following:
diff --git a/docs/index.rst b/docs/index.rst
@@ -31,14 +31,16 @@ You can always find the latest release `here <https://github.com/eth-cscs/refram
 Publications
 ============
 
-* Slides [`pdf <https://drive.google.com/open?id=1iwg1I48LVaWhhZCZIYPJSi3hdFLRcuhi>`__] @ `HPC Knowledge Meeting '19 <https://hpckp.org/>`__
-* Slides [`pdf <https://fosdem.org/2019/schedule/event/reframe/attachments/slides/3226/export/events/attachments/reframe/slides/3226/FOSDEM_2019.pdf>`__] & `Talk <https://fosdem.org/2019/schedule/event/reframe/>`__ @ `FOSDEM'19 <https://fosdem.org/2019/>`__
-* Slides [`pdf <https://indico.cism.ucl.ac.be/event/4/contributions/24/attachments/30/62/ReFrame_EUM_2019.pdf>`__] @ `4th EasyBuild User Meeting <https://github.com/easybuilders/easybuild/wiki/4th-EasyBuild-User-Meeting>`__
-* Slides [`pdf <https://drive.google.com/open?id=1bSykDrl1e2gPflf4jFJ8kfe_SZAtrJ_Q>`__] @ `SC18 <https://sc18.supercomputing.org/>`__
-* Slides [`pdf <https://github.com/eth-cscs/UserLabDay/blob/master/slides/ci_and_regression/ReFrame_CI.pdf>`__] [`pptx <https://github.com/eth-cscs/UserLabDay/blob/master/slides/ci_and_regression/ReFrame_CI.pptx>`__] @ `CSCS User Lab Day 2018 <https://github.com/eth-cscs/UserLabDay>`__
-* Slides [`pdf <https://drive.google.com/open?id=1sZhibvUlGlT670aOHPdMlWFffWptYzLX>`__] @ `HPC Advisory Council 2018 <http://www.hpcadvisorycouncil.com/events/2018/swiss-workshop/>`__
-* Slides [`pdf <https://drive.google.com/open?id=1EyJ-siupkgLeVT54A4WlFpQtrJaU0xOy>`__] @ `SC17 <https://sc17.supercomputing.org/>`__
-* Slides [`pdf <https://drive.google.com/open?id=18VrCy0MTplGo67uxVbzYZicQChor9VSY>`__] @ `CUG 2017 <https://cug.org/cug-2017/>`__
+* Slides [`pdf <https://drive.google.com/open?id=1Z3faPh9OSSXvlLHL07co3MRRn443dYsY>`__] @ `HPC System Testing BoF <https://sc19.supercomputing.org/session/?sess=sess324>`__, SC'19.
+* Slides [`pdf <https://drive.google.com/open?id=1JOFqY3ejbR1X5kTn_IZyp1GlCd2ZZS58>`__] @ `HUST 2019 <https://sc19.supercomputing.org/session/?sess=sess116>`__, SC'19.
+* Slides [`pdf <https://drive.google.com/open?id=1iwg1I48LVaWhhZCZIYPJSi3hdFLRcuhi>`__] @ `HPC Knowledge Meeting '19 <https://hpckp.org/>`__.
+* Slides [`pdf <https://fosdem.org/2019/schedule/event/reframe/attachments/slides/3226/export/events/attachments/reframe/slides/3226/FOSDEM_2019.pdf>`__] & `Talk <https://fosdem.org/2019/schedule/event/reframe/>`__ @ `FOSDEM'19 <https://fosdem.org/2019/>`__.
+* Slides [`pdf <https://indico.cism.ucl.ac.be/event/4/contributions/24/attachments/30/62/ReFrame_EUM_2019.pdf>`__] @ `4th EasyBuild User Meeting <https://github.com/easybuilders/easybuild/wiki/4th-EasyBuild-User-Meeting>`__.
+* Slides [`pdf <https://drive.google.com/open?id=1bSykDrl1e2gPflf4jFJ8kfe_SZAtrJ_Q>`__] @ `HUST 2018 <https://sc18.supercomputing.org/>`__, SC'18.
+* Slides [`pdf <https://github.com/eth-cscs/UserLabDay/blob/master/slides/ci_and_regression/ReFrame_CI.pdf>`__] [`pptx <https://github.com/eth-cscs/UserLabDay/blob/master/slides/ci_and_regression/ReFrame_CI.pptx>`__] @ `CSCS User Lab Day 2018 <https://github.com/eth-cscs/UserLabDay>`__.
+* Slides [`pdf <https://drive.google.com/open?id=1sZhibvUlGlT670aOHPdMlWFffWptYzLX>`__] @ `HPC Advisory Council 2018 <http://www.hpcadvisorycouncil.com/events/2018/swiss-workshop/>`__.
+* Slides [`pdf <https://drive.google.com/open?id=1EyJ-siupkgLeVT54A4WlFpQtrJaU0xOy>`__] @ `SC17 <https://sc17.supercomputing.org/>`__.
+* Slides [`pdf <https://drive.google.com/open?id=18VrCy0MTplGo67uxVbzYZicQChor9VSY>`__] @ `CUG 2017 <https://cug.org/cug-2017/>`__.
 
 
 
diff --git a/reframe/__init__.py b/reframe/__init__.py
@@ -2,7 +2,7 @@
 import sys
 
 
-VERSION = '2.21-dev2'
+VERSION = '2.21-dev3'
 INSTALL_PREFIX = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 MIN_PYTHON_VERSION = (3, 5, 0)
 
diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py
@@ -11,13 +11,16 @@ class JobLauncher(abc.ABC):
     program to multiple nodes, e.g., ``mpirun``, ``srun`` etc.
 
     .. note::
-       This is an abstract class.
-       Regression tests may not instantiate this class directly.
+
+       Users cannot create job launchers directly. You may retrieve a
+       registered launcher backend through the
+       :func:`reframe.core.launchers.registry.getlauncher` function.
 
     .. note::
        .. versionchanged:: 2.8
           Job launchers do not get a reference to a job during their
           initialization.
+
     '''
 
     #: List of options to be passed to the job launcher invocation.
@@ -26,20 +29,13 @@ class JobLauncher(abc.ABC):
     #: :default: ``[]``
     options = fields.TypedField('options', typ.List[str])
 
-    def __init__(self, options=[]):
-        self.options = list(options)
+    def __init__(self):
+        self.options = []
 
     @abc.abstractmethod
     def command(self, job):
-        '''The launcher command.
-
-        :arg job: A :class:`reframe.core.schedulers.Job` that will be used by
-            this launcher to properly emit its options.
-            Subclasses may override this method and emit options according the
-            number of tasks associated to the job etc.
-        :returns: a list of command line arguments (including the launcher
-            executable).
-        '''
+        # The launcher command to be emitted for ``job``
+        pass
 
     def run_command(self, job):
         return ' '.join(self.command(job) + self.options)
@@ -48,15 +44,15 @@ def run_command(self, job):
 class LauncherWrapper(JobLauncher):
     '''Wrap a launcher object so as to modify its invocation.
 
-    This is useful for parallel debuggers.
-    For example, to launch a regression test using the `DDT
-    <https://www.allinea.com/products/ddt/>`_ debugger, you can do the
-    following:
+    This is useful for parallel debuggers. For example, to launch a regression
+    test using the `ARM DDT
+    <https://www.arm.com/products/development-tools/server-and-hpc/forge>`__
+    debugger, you can do the following:
 
-    ::
+    .. code:: python
 
-        def setup(self, partition, environ, **job_opts):
-            super().setup(partition, environ, **job_opts)
+        @rfm.run_after('setup')
+        def set_launcher(self):
             self.job.launcher = LauncherWrapper(self.job.launcher, 'ddt',
                                                 ['--offline'])
 
@@ -76,12 +72,14 @@ def setup(self, partition, environ, **job_opts):
     :arg target_launcher: The launcher to wrap.
     :arg wrapper_command: The wrapper command.
     :arg wrapper_options: List of options to pass to the wrapper command.
+
     '''
 
     def __init__(self, target_launcher, wrapper_command, wrapper_options=[]):
-        super().__init__(target_launcher.options)
+        super().__init__()
+        self.options = target_launcher.options
         self._target_launcher = target_launcher
-        self._wrapper_command = [wrapper_command] + list(wrapper_options)
+        self._wrapper_command = [wrapper_command] + wrapper_options
 
     def command(self, job):
         return self._wrapper_command + self._target_launcher.command(job)
diff --git a/reframe/core/launchers/local.py b/reframe/core/launchers/local.py
@@ -5,9 +5,10 @@
 
 @register_launcher('local', local=True)
 class LocalLauncher(JobLauncher):
-    def __init__(self, options=[]):
-        # Ignore options passed by users
-        super().__init__([])
-
     def command(self, job):
+        # Reset any options set by the user
+        #
+        # NOTE: This assumes that the `command` is called before accessing
+        # `self.options`.
+        self.options = []
         return []
diff --git a/reframe/core/launchers/mpi.py b/reframe/core/launchers/mpi.py
@@ -19,7 +19,17 @@ def command(self, job):
 @register_launcher('alps')
 class AlpsLauncher(JobLauncher):
     def command(self, job):
-        return ['aprun', '-B']
+        cmd = ['aprun', '-n', str(job.num_tasks)]
+        if job.num_tasks_per_node:
+            cmd += ['-N', str(job.num_tasks_per_node)]
+
+        if job.num_cpus_per_task:
+            cmd += ['-d', str(job.num_cpus_per_task)]
+
+        if job.use_smt:
+            cmd += ['-j', '0']
+
+        return cmd
 
 
 @register_launcher('mpirun')
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
@@ -972,13 +972,13 @@ def _setup_job(self, **job_opts):
             scheduler_type = self._current_partition.scheduler
             launcher_type = self._current_partition.launcher
 
-        self._job = scheduler_type(
-            name='rfm_%s_job' % self.name,
-            launcher=launcher_type(),
-            workdir=self._stagedir,
-            sched_access=self._current_partition.access,
-            sched_exclusive_access=self.exclusive_access,
-            **job_opts)
+        self._job = Job.create(scheduler_type(),
+                               launcher_type(),
+                               name='rfm_%s_job' % self.name,
+                               workdir=self._stagedir,
+                               sched_access=self._current_partition.access,
+                               sched_exclusive_access=self.exclusive_access,
+                               **job_opts)
 
         # Get job options from managed resources and prepend them to
         # job_opts. We want any user supplied options to be able to
@@ -1101,11 +1101,10 @@ def compile(self):
         environs = [self._current_partition.local_env, self._current_environ,
                     user_environ, self._cdt_environ]
 
-        self._build_job = getscheduler('local')(
-            name='rfm_%s_build' % self.name,
-            launcher=getlauncher('local')(),
-            workdir=self._stagedir)
-
+        self._build_job = Job.create(getscheduler('local')(),
+                                     launcher=getlauncher('local')(),
+                                     name='rfm_%s_build' % self.name,
+                                     workdir=self._stagedir)
         with os_ext.change_dir(self._stagedir):
             try:
                 self._build_job.prepare(build_commands, environs,
diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py
diff --git a/reframe/core/schedulers/local.py b/reframe/core/schedulers/local.py
diff --git a/reframe/core/schedulers/pbs.py b/reframe/core/schedulers/pbs.py
diff --git a/reframe/core/schedulers/slurm.py b/reframe/core/schedulers/slurm.py
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py
diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ run_tutorial_checks()`
`50`	`50`	`run_user_checks()`
`51`	`51`	`{`
`52`	`52`	`cmd="./bin/reframe -C config/cscs.py --exec-policy=async --save-log-files \`
`53`		`--r --flex-alloc-tasks=2 -t production\|benchmark $@"`
	`53`	`+-r --flex-alloc-nodes=2 -t production\|benchmark $@"`
`54`	`54`	echo "Running user checks with \`$cmd'"
`55`	`55`	`checked_exec $cmd`
`56`	`56`	`}`