Merge branch 'master' into docs/syntax

Author: Vasileios Karakasis

.github/workflows/main.yml

@@ -47,6 +47,17 @@ jobs:
       run: |
         docker run reframe:${{ matrix.modules-version }}
 
+  tutorialtest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build Image for Tutorial Tests
+      run: |
+        docker build -f ci-scripts/dockerfiles/tutorials.dockerfile -t reframe:tutorials .
+    - name: Run Tutorial Tests
+      run: |
+        docker run reframe:tutorials
+
   unusedimports:
     runs-on: ubuntu-latest
     steps:

ci-scripts/dockerfiles/tutorials.dockerfile

@@ -0,0 +1,40 @@
+#
+# Execute this from the top-level ReFrame source directory
+#
+
+
+FROM reframehpc/rfm-ci-base:lmod
+
+ENV _SPACK_VER=0.16
+ENV _EB_VER=4.4.1
+
+# Required utilities
+RUN apt-get -y update && \
+    apt-get -y install curl
+
+# ReFrame user
+RUN useradd -ms /bin/bash rfmuser
+
+USER rfmuser
+
+# Install Spack
+RUN git clone https://github.com/spack/spack ~/spack && \
+    cd ~/spack && \
+    git checkout releases/v${_SPACK_VER}
+
+RUN pip3 install easybuild==${_EB_VER}
+
+ENV PATH="/home/rfmuser/.local/bin:${PATH}"
+
+# Install ReFrame from the current directory
+COPY --chown=rfmuser . /home/rfmuser/reframe/
+
+WORKDIR /home/rfmuser/reframe
+
+RUN ./bootstrap.sh
+
+RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/rfmuser/spack/share/spack/setup-env.sh' > /home/rfmuser/setup.sh
+
+ENV BASH_ENV /home/rfmuser/setup.sh
+
+CMD ["/bin/bash", "-c", "./bin/reframe -r -C tutorials/config/settings.py -R -c tutorials/build_systems --system tutorials-docker"]

cscs-checks/apps/jupyter/check_ipcmagic.py

@@ -11,55 +11,79 @@
 
 @rfm.simple_test
 class IPCMagicCheck(rfm.RunOnlyRegressionTest):
-    def __init__(self):
-        self.descr = 'Distributed training with TensorFlow using ipyparallel'
-        self.valid_systems = ['daint:gpu', 'dom:gpu']
-        self.valid_prog_environs = ['PrgEnv-gnu']
-        self.modules = [
-            # FIXME: Use the default ipcmagic version when fixed
-            f'ipcmagic/0.1-CrayGNU-{osext.cray_cdt_version()}',
-            f'Horovod/0.21.0-CrayGNU-{osext.cray_cdt_version()}-tf-2.4.0'
-        ]
-        self.num_tasks = 2
-        self.num_tasks_per_node = 1
-        self.executable = 'ipython'
-        self.executable_opts = ['tf-hvd-sgd-ipc-tf2.py']
-        nids = sn.extractall(r'nid(?P<nid>\d+)',
-                             self.stdout, 'nid', str)
-        self.sanity_patterns = sn.all([
-            sn.assert_ne(nids, []),
-            sn.assert_ne(nids[0], nids[1])
-        ])
-        self.reference = {
-            'daint:gpu': {
-                'slope': (2.0, -0.1, 0.1, None),
-                'offset': (0.0, -0.1, 0.1, None),
-                'retries': (0, None, None, None),
-                'time': (10, None, None, 's'),
-            },
-            'dom:gpu': {
-                'slope': (2.0, -0.1, 0.1, None),
-                'offset': (0.0, -0.1, 0.1, None),
-                'retries': (0, None, None, None),
-                'time': (10, None, None, 's'),
-            }
-        }
-        self.perf_patterns = {
-            'slope': sn.extractsingle(r'slope=(?P<slope>\S+)',
-                                      self.stdout, 'slope', float),
-            'offset': sn.extractsingle(r'offset=(?P<offset>\S+)',
-                                       self.stdout, 'offset', float),
-            'retries': 4 - sn.count(sn.findall(r'IPCluster is already running',
-                                               self.stdout)),
-            'time': sn.extractsingle(r'IPCluster is ready\!\s+'
-                                     r'\((?P<time>\d+) seconds\)',
-                                     self.stdout, 'time', float)
+    descr = 'Distributed training with TensorFlow using ipyparallel'
+    valid_systems = ['daint:gpu', 'dom:gpu']
+    valid_prog_environs = ['PrgEnv-gnu']
+    modules = [
+        f'ipcmagic', f'jupyterlab',
+        f'Horovod/0.21.0-CrayGNU-{osext.cray_cdt_version()}-tf-2.4.0'
+    ]
+    num_tasks = 2
+    num_tasks_per_node = 1
+    executable = 'ipython'
+    executable_opts = ['tf-hvd-sgd-ipc-tf2.py']
+    reference = {
+        'daint:gpu': {
+            'slope': (2.0, -0.1, 0.1, 'N/A'),
+            'offset': (0.0, -0.1, 0.1, 'N/A'),
+            'retries': (0, None, None, 'N/A'),
+            'time': (10, None, None, 's'),
+        },
+        'dom:gpu': {
+            'slope': (2.0, -0.1, 0.1, 'N/A'),
+            'offset': (0.0, -0.1, 0.1, 'N/A'),
+            'retries': (0, None, None, 'N/A'),
+            'time': (10, None, None, 's'),
         }
-        self.maintainers = ['RS', 'TR']
-        self.tags = {'production'}
+    }
+
+    maintainers = ['RS', 'TR']
+    tags = {'production'}
+
+    @run_after('setup')
+    def daint_module_workaround(self):
+        if self.current_system.name == 'daint':
+            # FIXME: Use the default modules once Dom/Daint are aligned
+            self.modules = [
+                f'ipcmagic/1.0.1-CrayGNU-{osext.cray_cdt_version()}',
+                f'Horovod/0.21.0-CrayGNU-{osext.cray_cdt_version()}-tf-2.4.0'
+            ]
+            # FIXME: Enforce loading of jupyterlab module since
+            # `module show jupyterlab` throws a Tcl error on Daint
+            self.prerun_cmds = ['module load jupyterlab']
+
+    @sanity_function
+    def assert_successful_execution(self):
+        nids = sn.extractall(r'nid(?P<nid>\d+)', self.stdout, 'nid', str)
+        return sn.all([
+            sn.assert_ne(nids, []), sn.assert_ne(nids[0], nids[1]),
+            sn.assert_found(r'IPCluster is ready\!\s+', self.stdout),
+            sn.assert_found(r'slope=\S+', self.stdout)
+        ])
+
+    @performance_function('N/A')
+    def slope(self):
+        return sn.extractsingle(r'slope=(?P<slope>\S+)', self.stdout,
+                                'slope', float)
+
+    @performance_function('N/A')
+    def offset(self):
+        return sn.extractsingle(r'offset=(?P<offset>\S+)', self.stdout,
+                                'offset', float)
+
+    @performance_function('N/A')
+    def retries(self):
+        return 4 - sn.count(sn.findall(r'IPCluster is already running',
+                                       self.stdout))
+
+    @performance_function('s')
+    def time(self):
+        return sn.extractsingle(r'IPCluster is ready\!\s+'
+                                r'\((?P<time>\d+) seconds\)',
+                                self.stdout, 'time', float)
 
     @run_before('run')
-    def prepare_run(self):
+    def reset_launcher(self):
         # Change the job launcher since `ipython`
         # needs to be launched without `srun`.
         self.job.launcher = getlauncher('local')()

cscs-checks/apps/namd/namd_check.py

@@ -9,24 +9,62 @@
 import reframe.utility.sanity as sn
 
 
-class NamdBaseCheck(rfm.RunOnlyRegressionTest):
-    def __init__(self, arch, scale, variant):
-        self.descr = f'NAMD check ({arch}, {variant})'
-        if self.current_system.name in ['eiger', 'pilatus']:
-            self.valid_prog_environs = ['cpeGNU']
-        else:
-            self.valid_prog_environs = ['builtin']
+@rfm.simple_test
+class NamdCheck(rfm.RunOnlyRegressionTest):
+    scale = parameter(['small', 'large'])
+    variant = parameter(['maint', 'prod'])
+    arch = parameter(['gpu', 'cpu'])
 
-        self.modules = ['NAMD']
+    valid_prog_environs = ['builtin', 'cpeGNU']
+    modules = ['NAMD']
+    executable = 'namd2'
+    use_multithreading = True
+    num_tasks_per_core = 2
+    maintainers = ['CB', 'LM']
+    tags = {'scs', 'external-resources'}
+    extra_resources = {
+        'switches': {
+            'num_switches': 1
+        }
+    }
 
-        # Reset sources dir relative to the SCS apps prefix
-        self.sourcesdir = os.path.join(self.current_system.resourcesdir,
-                                       'NAMD', 'prod')
-        self.executable = 'namd2'
-        self.use_multithreading = True
-        self.num_tasks_per_core = 2
+    @run_after('init')
+    def adapt_description(self):
+        self.descr = f'NAMD check ({self.arch}, {self.variant})'
+        self.tags |= {
+            'maintenance' if self.variant == 'maint' else 'production'
+        }
+
+    @run_after('init')
+    def adapt_valid_systems(self):
+        if self.arch == 'gpu':
+            self.valid_systems = ['daint:gpu']
+            if self.scale == 'small':
+                self.valid_systems += ['dom:gpu']
+        else:
+            self.valid_systems = ['daint:mc', 'eiger:mc', 'pilatus:mc']
+            if self.scale == 'small':
+                self.valid_systems += ['dom:mc']
+
+    @run_after('init')
+    def adapt_valid_prog_environs(self):
+        if self.current_system.name in ['eiger', 'pilatus']:
+            self.valid_prog_environs.remove('builtin')
 
-        if scale == 'small':
+    @run_after('init')
+    def setup_parallel_run(self):
+        if self.arch == 'gpu':
+            self.executable_opts = ['+idlepoll', '+ppn 23', 'stmv.namd']
+            self.num_cpus_per_task = 24
+            self.num_gpus_per_node = 1
+        else:
+            # On Eiger a no-smp NAMD version is the default
+            if self.current_system.name in ['eiger', 'pilatus']:
+                self.executable_opts = ['+idlepoll', 'stmv.namd']
+            else:
+                self.executable_opts = ['+idlepoll', '+ppn 71', 'stmv.namd']
+                self.num_cpus_per_task = 72
+        if self.scale == 'small':
             # On Eiger a no-smp NAMD version is the default
             if self.current_system.name in ['eiger', 'pilatus']:
                 self.num_tasks = 768
@@ -42,85 +80,57 @@ def __init__(self, arch, scale, variant):
                 self.num_tasks = 16
                 self.num_tasks_per_node = 1
 
+    @run_before('compile')
+    def prepare_build(self):
+        # Reset sources dir relative to the SCS apps prefix
+        self.sourcesdir = os.path.join(self.current_system.resourcesdir,
+                                       'NAMD', 'prod')
+
+    @sanity_function
+    def validate_energy(self):
         energy = sn.avg(sn.extractall(
             r'ENERGY:([ \t]+\S+){10}[ \t]+(?P<energy>\S+)',
             self.stdout, 'energy', float)
         )
         energy_reference = -2451359.5
         energy_diff = sn.abs(energy - energy_reference)
-        self.sanity_patterns = sn.all([
+        return sn.all([
             sn.assert_eq(sn.count(sn.extractall(
                          r'TIMING: (?P<step_num>\S+)  CPU:',
                          self.stdout, 'step_num')), 50),
             sn.assert_lt(energy_diff, 2720)
         ])
 
-        self.perf_patterns = {
-            'days_ns': sn.avg(sn.extractall(
-                r'Info: Benchmark time: \S+ CPUs \S+ '
-                r's/step (?P<days_ns>\S+) days/ns \S+ MB memory',
-                self.stdout, 'days_ns', float))
-        }
-
-        self.maintainers = ['CB', 'LM']
-        self.tags = {'scs', 'external-resources'}
-        self.extra_resources = {
-            'switches': {
-                'num_switches': 1
-            }
-        }
-
-
-@rfm.parameterized_test(*([s, v]
-                          for s in ['small', 'large']
-                          for v in ['maint', 'prod']))
-class NamdGPUCheck(NamdBaseCheck):
-    def __init__(self, scale, variant):
-        super().__init__('gpu', scale, variant)
-        self.valid_systems = ['daint:gpu']
-        self.executable_opts = ['+idlepoll', '+ppn 23', 'stmv.namd']
-        self.num_cpus_per_task = 24
-        self.num_gpus_per_node = 1
-        self.tags |= {'maintenance' if variant == 'maint' else 'production'}
-        if scale == 'small':
-            self.valid_systems += ['dom:gpu']
-            self.reference = {
-                'dom:gpu': {'days_ns': (0.15, None, 0.05, 'days/ns')},
-                'daint:gpu': {'days_ns': (0.15, None, 0.05, 'days/ns')}
-            }
-        else:
-            self.reference = {
-                'daint:gpu': {'days_ns': (0.07, None, 0.05, 'days/ns')}
-            }
-
-
-@rfm.parameterized_test(*([s, v]
-                          for s in ['small', 'large']
-                          for v in ['maint', 'prod']))
-class NamdCPUCheck(NamdBaseCheck):
-    def __init__(self, scale, variant):
-        super().__init__('cpu', scale, variant)
-        self.valid_systems = ['daint:mc', 'eiger:mc', 'pilatus:mc']
-        # On Eiger a no-smp NAMD version is the default
-        if self.current_system.name in ['eiger', 'pilatus']:
-            self.executable_opts = ['+idlepoll', 'stmv.namd']
-            self.num_tasks_per_core = 2
-        else:
-            self.executable_opts = ['+idlepoll', '+ppn 71', 'stmv.namd']
-            self.num_cpus_per_task = 72
-        if scale == 'small':
-            self.valid_systems += ['dom:mc']
-            self.reference = {
-                'dom:mc': {'days_ns': (0.51, None, 0.05, 'days/ns')},
-                'daint:mc': {'days_ns': (0.51, None, 0.05, 'days/ns')},
-                'eiger:mc': {'days_ns': (0.12, None, 0.05, 'days/ns')},
-                'pilatus:mc': {'days_ns': (0.12, None, 0.05, 'days/ns')},
-            }
+    @run_before('performance')
+    def set_reference(self):
+        if self.arch == 'gpu':
+            if self.scale == 'small':
+                self.reference = {
+                    'dom:gpu': {'days_ns': (0.15, None, 0.05, 'days/ns')},
+                    'daint:gpu': {'days_ns': (0.15, None, 0.05, 'days/ns')}
+                }
+            else:
+                self.reference = {
+                    'daint:gpu': {'days_ns': (0.07, None, 0.05, 'days/ns')}
+                }
         else:
-            self.reference = {
-                'daint:mc': {'days_ns': (0.28, None, 0.05, 'days/ns')},
-                'eiger:mc': {'days_ns': (0.05, None, 0.05, 'days/ns')},
-                'pilatus:mc': {'days_ns': (0.05, None, 0.05, 'days/ns')}
-            }
+            if self.scale == 'small':
+                self.reference = {
+                    'dom:mc': {'days_ns': (0.51, None, 0.05, 'days/ns')},
+                    'daint:mc': {'days_ns': (0.51, None, 0.05, 'days/ns')},
+                    'eiger:mc': {'days_ns': (0.12, None, 0.05, 'days/ns')},
+                    'pilatus:mc': {'days_ns': (0.12, None, 0.05, 'days/ns')},
+                }
+            else:
+                self.reference = {
+                    'daint:mc': {'days_ns': (0.28, None, 0.05, 'days/ns')},
+                    'eiger:mc': {'days_ns': (0.05, None, 0.05, 'days/ns')},
+                    'pilatus:mc': {'days_ns': (0.05, None, 0.05, 'days/ns')}
+                }
 
-        self.tags |= {'maintenance' if variant == 'maint' else 'production'}
+    @performance_function('days/ns')
+    def days_ns(self):
+        return sn.avg(sn.extractall(
+            r'Info: Benchmark time: \S+ CPUs \S+ '
+            r's/step (?P<days_ns>\S+) days/ns \S+ MB memory',
+            self.stdout, 'days_ns', float))

cscs-checks/apps/quantumespresso/quantumespresso_check.py

@@ -19,7 +19,8 @@ def __init__(self):
 
         self.modules = ['QuantumESPRESSO']
         self.executable = 'pw.x'
-        self.executable_opts = ['-in', 'ausurf.in']
+        self.executable_opts = ['-in', 'ausurf.in',
+                                '-pd', '.true.']
 
         self.sanity_patterns = sn.all([
             sn.assert_found(r'convergence has been achieved', self.stdout),