Merge branch 'master' into test/dgemm

Author: Vasileios Karakasis

cscs-checks/apps/spark/spark_check.py

@@ -12,21 +12,22 @@
 
 @rfm.simple_test
 class SparkCheck(rfm.RunOnlyRegressionTest):
+    variant = parameter(['spark', 'pyspark'])
+
     def __init__(self):
-        self.descr = 'Simple calculation of pi with Spark'
+        self.descr = f'Simple calculation of pi with {self.variant}'
         self.valid_systems = ['daint:gpu', 'daint:mc',
                               'dom:gpu', 'dom:mc']
         self.valid_prog_environs = ['builtin']
         self.modules = ['Spark']
-        self.sourcesdir = None
         self.prerun_cmds = ['start-all.sh']
         self.postrun_cmds = ['stop-all.sh']
-        self.num_tasks = 2
+        self.num_tasks = 3
         self.num_tasks_per_node = 1
         pi_value = sn.extractsingle(r'Pi is roughly\s+(?P<pi>\S+)',
                                     self.stdout, 'pi', float)
         self.sanity_patterns = sn.assert_lt(sn.abs(pi_value - math.pi), 0.01)
-        self.maintainers = ['TM', 'TR']
+        self.maintainers = ['TM', 'RS']
         self.tags = {'production'}
 
     @rfm.run_before('run')
@@ -39,16 +40,24 @@ def prepare_run(self):
             exec_cores = 9
 
         self.variables = {
-            'SPARK_WORKER_CORES': '%s' % num_workers,
+            'SPARK_WORKER_CORES': str(num_workers),
             'SPARK_LOCAL_DIRS': '"/tmp"',
         }
-        self.executable = (
-            f'spark-submit --conf spark.default.parallelism={num_workers} '
-            f'--conf spark.executor.cores={exec_cores} '
-            f'--conf spark.executor.memory=15g --master $SPARKURL '
-            f'--class org.apache.spark.examples.SparkPi '
-            f'$EBROOTSPARK/examples/jars/spark-examples*.jar 10000;'
-        )
+        self.executable = 'spark-submit'
+        self.executable_opts = [
+            f'--conf spark.default.parallelism={num_workers}',
+            f'--conf spark.executor.cores={exec_cores}',
+            f'--conf spark.executor.memory=15g',
+            f'--master $SPARKURL'
+        ]
+        if self.variant == 'spark':
+            self.executable_opts += [
+                '--class org.apache.spark.examples.SparkPi',
+                '$EBROOTSPARK/examples/jars/spark-examples*.jar 10000'
+            ]
+        else:
+            self.executable_opts.append('spark_pi.py')
+
         # The job launcher has to be changed since the `spark-submit`
         # script is not used with srun.
         self.job.launcher = getlauncher('local')()

cscs-checks/apps/spark/src/spark_pi.py

@@ -0,0 +1,17 @@
+import random
+from pyspark import SparkContext, SparkConf
+
+
+conf = SparkConf().setAppName('pyspark')
+sc = SparkContext(conf=conf)
+NUM_SAMPLES = 10000000
+
+
+def inside(p):
+    x, y = random.random(), random.random()
+    return x*x + y*y < 1
+
+
+if __name__ == '__main__':
+    count = sc.parallelize(range(0, NUM_SAMPLES)).filter(inside).count()
+    print('Pi is roughly %f' % (4.0 * count / NUM_SAMPLES))

cscs-checks/system/io/ior_check.py

@@ -23,6 +23,12 @@ def __init__(self, base_dir):
         self.prerun_cmds = ['mkdir -p ' + self.test_dir]
         self.test_file = os.path.join(self.test_dir, 'ior')
         self.fs = {
+            '/scratch/e1000': {
+                'valid_systems': ['eiger:mc'],
+                'eiger': {
+                    'num_tasks': 10,
+                }
+            },
             '/scratch/snx3000tds': {
                 'valid_systems': ['dom:gpu', 'dom:mc'],
                 'dom': {
@@ -105,7 +111,7 @@ def __init__(self, base_dir):
 
         self.maintainers = ['SO', 'GLR']
 
-        systems_to_test = ['dom', 'daint']
+        systems_to_test = ['dom', 'daint', 'eiger']
         if self.current_system.name in systems_to_test:
             self.tags |= {'production', 'external-resources'}
 
@@ -115,7 +121,8 @@ def set_exec_opts(self):
         self.executable_opts += ['-o', self.test_file]
 
 
-@rfm.parameterized_test(['/scratch/snx3000tds'],
+@rfm.parameterized_test(['/scratch/e1000'],
+                        ['/scratch/snx3000tds'],
                         ['/scratch/snx3000'],
                         ['/users'],
                         ['/scratch/shared/fulen'])
@@ -132,7 +139,8 @@ def __init__(self, base_dir):
         self.tags |= {'write'}
 
 
-@rfm.parameterized_test(['/scratch/snx3000tds'],
+@rfm.parameterized_test(['/scratch/e1000'],
+                        ['/scratch/snx3000tds'],
                         ['/scratch/snx3000'],
                         ['/users'],
                         ['/scratch/shared/fulen'])

docs/Makefile

@@ -8,7 +8,6 @@ SPHINXOPTS  =
 SPHINXBUILD =  -msphinx
 SPHINXPROJ  = ReFrame
 SOURCEDIR   = .
-BUILDDIR    = $(VERSION)
 RM          = /bin/rm -rf
 
 TARGET_DOCS := \
@@ -48,7 +47,7 @@ clean:
 	-$(RM) $(TARGET_DOCS) doctrees
 
 $(TARGET_DOCS): Makefile
-	@$(PYTHON) $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(PYTHON) $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "" $(SPHINXOPTS) $(O)
 
 
 

docs/config_reference.rst

@@ -308,6 +308,16 @@ System Partition Configuration
    This option is relevant only when ReFrame executes with the `asynchronous execution policy <pipeline.html#execution-policies>`__.
 
 
+.. js:attribute:: .systems[].partitions[].prepare_cmds
+
+   :required: No
+   :default: ``[]``
+
+   List of shell commands to be emitted before any environment loading commands are emitted.
+
+   .. versionadded:: 3.5.0
+
+
 .. js:attribute:: .systems[].partitions[].resources
 
    :required: No

docs/tutorial_advanced.rst

@@ -389,13 +389,16 @@ Generally, ReFrame generates the job shell scripts using the following pattern:
 
    #!/bin/bash -l
    {job_scheduler_preamble}
-   {test_environment}
+   {prepare_cmds}
+   {env_load_cmds}
    {prerun_cmds}
    {parallel_launcher} {executable} {executable_opts}
    {postrun_cmds}
 
 The ``job_scheduler_preamble`` contains the backend job scheduler directives that control the job allocation.
-The ``test_environment`` are the necessary commands for setting up the environment of the test.
+The ``prepare_cmds`` are commands that can be emitted before the test environment commands.
+These can be specified with the :js:attr:`prepare_cmds <.systems[].partitions[].prepare_cmds>` partition configuration option.
+The ``env_load_cmds`` are the necessary commands for setting up the environment of the test.
 These include any modules or environment variables set at the `system partition level <config_reference.html#system-partition-configuration>`__ or any `modules <regression_test_api.html#reframe.core.pipeline.RegressionTest.modules>`__ or `environment variables <regression_test_api.html#reframe.core.pipeline.RegressionTest.variables>`__ set at the test level.
 Then the commands specified in :attr:`prerun_cmds <reframe.core.pipeline.RegressionTest.prerun_cmds>` follow, while those specified in the :attr:`postrun_cmds <reframe.core.pipeline.RegressionTest.postrun_cmds>` come after the launch of the parallel job.
 The parallel launch itself consists of three parts:
@@ -671,13 +674,14 @@ ReFrame can be used also to test applications that run inside a container.
 First, we need to enable the container platform support in ReFrame's configuration and, specifically, at the partition configuration level:
 
 .. literalinclude:: ../tutorials/config/settings.py
-   :lines: 38-58
-   :emphasize-lines: 15-20
+   :lines: 38-62
+   :emphasize-lines: 15-24
 
 For each partition, users can define a list of container platforms supported using the :js:attr:`container_platforms` `configuration parameter <config_reference.html#.systems[].partitions[].container_platforms>`__.
-In this case, we define the `Singularity <https://sylabs.io>`__ platform, for which we set the :js:attr:`modules` parameter in order to instruct ReFrame to load the ``singularity`` module, whenever it needs to run with this container platform.
+In this case, we define the `Sarus <https://github.com/eth-cscs/sarus>`__ platform for which we set the :js:attr:`modules` parameter in order to instruct ReFrame to load the ``sarus`` module, whenever it needs to run with this container platform.
+Similarly, we add an entry for the `Singularity <https://sylabs.io>`__ platform.
 
-The following test will use a Singularity container to run:
+The following parameterized test, will create two tests, one for each of the supported container platforms:
 
 .. code-block:: console
 
@@ -690,29 +694,66 @@ The following test will use a Singularity container to run:
 
 A container-based test can be written as :class:`RunOnlyRegressionTest <reframe.core.pipeline.RunOnlyRegressionTest>` that sets the :attr:`container_platform <reframe.core.pipeline.RegressionTest.container_platform>` attribute.
 This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test.
-In this case, the test will be using `Singularity <https://sylabs.io>`__ as a container platform.
 If such a platform is not `configured <config_reference.html#container-platform-configuration>`__ for the current system, the test will fail.
 
-As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container by setting the :attr:`image <reframe.core.containers.ContainerPlatform.image>` and the :attr:`commands <reframe.core.containers.ContainerPlatform.commands>` container platform attributes.
-These two attributes are mandatory for container-based checks.
+As soon as the container platform to be used is defined, you need to specify the container image to use by setting the :attr:`image <reframe.core.containers.ContainerPlatform.image>`.
+In the ``Singularity`` test variant, we add the ``docker://`` prefix to the image name, in order to instruct ``Singularity`` to pull the image from `DockerHub <https://hub.docker.com/>`__.
+The default command that the container runs can be overwritten by setting the :attr:`command <reframe.core.containers.ContainerPlatform.command>` attribute of the container platform.
+
+The :attr:`image <reframe.core.containers.ContainerPlatform.image>` is the only mandatory attribute for container-based checks.
 It is important to note that the :attr:`executable <reframe.core.pipeline.RegressionTest.executable>` and :attr:`executable_opts <reframe.core.pipeline.RegressionTest.executable_opts>` attributes of the actual test are ignored in case of container-based tests.
 
-ReFrame will run the container as follows:
+ReFrame will run the container according to the given platform as follows:
+
+.. code-block:: bash
+
+    # Sarus
+    sarus run --mount=type=bind,source="/path/to/test/stagedir",destination="/rfm_workdir" ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'
+
+    # Singularity
+    singularity exec -B"/path/to/test/stagedir:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'
 
-.. code-block:: console
 
-    singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release'
+In the ``Sarus`` case, ReFrame will prepend the following command in order to pull the container image before running the container:
 
-By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory.
-The user commands are then run from that directory one after the other.
+.. code-block:: bash
+
+   sarus pull ubuntu:18.04
+
+
+This is the default behavior of ReFrame, which can be changed if pulling the image is not desired by setting the :attr:`pull_image <reframe.core.containers.ContainerPlatform.pull_image>` attribute to :class:`False`.
+By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container.
 Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks.
-Users may also change the default mount point of the stage directory by using :attr:`workdir <reframe.core.pipeline.RegressionTest.container_platform.workdir>` attribute:
 Besides the stage directory, additional mount points can be specified through the :attr:`mount_points <reframe.core.pipeline.RegressionTest.container_platform.mount_points>` attribute:
 
 .. code-block:: python
 
     self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'),
                                             ('/path/to/host/dir2', '/path/to/container/mount_point2')]
 
+
+The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy artifacts as needed.
+These artifacts will therefore be available inside the stage directory after the container execution finishes.
+This is very useful if the artifacts are needed for the sanity or performance checks.
+If the copy is not performed by the default container command, the user can override this command by settings the :attr:`command <reframe.core.containers.ContainerPlatform.command>` attribute such as to include the appropriate copy commands.
+In the current test, the output of the ``cat /etc/os-release`` is available both in the standard output as well as in the ``release.txt`` file, since we have used the command:
+
+.. code-block:: bash
+
+    bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'
+
+
+and ``/rfm_workdir`` corresponds to the stage directory on the host system.
+Therefore, the ``release.txt`` file can now be used in the subsequent sanity checks:
+
+.. code-block:: python
+
+   os_release_pattern = r'18.04.\d+ LTS \(Bionic Beaver\)'
+   self.sanity_patterns = sn.all([
+       sn.assert_found(os_release_pattern, 'release.txt'),
+       sn.assert_found(os_release_pattern, self.stdout)
+   ])
+
+
 For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide.
 On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`.

docs/tutorial_basics.rst

@@ -323,7 +323,7 @@ Note that you should *not* edit this configuration file in place.
 Here is how the new configuration file looks like with the needed additions highlighted:
 
 .. literalinclude:: ../tutorials/config/settings.py
-   :lines: 10-24,76-97,130-
+   :lines: 10-24,80-101,134-
    :emphasize-lines: 3-15,31-42
 
 Here we define a system named ``catalina`` that has one partition named ``default``.
@@ -807,7 +807,7 @@ Let's extend our configuration file for Piz Daint.
 
 
 .. literalinclude:: ../tutorials/config/settings.py
-   :lines: 10-45,58-66,73-
+   :lines: 10-45,62-70,77-
    :emphasize-lines: 16-48,70-101,114-120