Skip to content

Commit 85dca12

Browse files
author
Vasileios Karakasis
authored
Merge branch 'master' into feat/improve-error-messages
2 parents 6941139 + 7eef1a8 commit 85dca12

File tree

21 files changed

+283
-62
lines changed

21 files changed

+283
-62
lines changed

bootstrap.sh

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,12 @@
99
# Bootstrap script for running ReFrame from source
1010
#
1111

12-
BLUE='\033[0;34m'
13-
GREEN='\033[0;32m'
14-
YELLOW='\033[0;33m'
15-
NC='\033[0m'
12+
if [ -t 1 ]; then
13+
BLUE='\033[0;34m'
14+
GREEN='\033[0;32m'
15+
YELLOW='\033[0;33m'
16+
NC='\033[0m'
17+
fi
1618

1719
CMD()
1820
{
@@ -70,15 +72,16 @@ fi
7072
# Check if ensurepip is installed
7173
$python -m ensurepip --version &> /dev/null
7274

75+
export PATH=$(pwd)/external/usr/bin:$PATH
76+
7377
# Install pip for Python 3
7478
if [ $? -eq 0 ]; then
75-
CMD $python -m ensurepip --root external/ --default-pip
79+
CMD $python -m ensurepip --root $(pwd)/external/ --default-pip
7680
fi
7781

78-
# ensurepip installs pip in `external/usr/` whereas the --target option installs
82+
# ensurepip installs pip in `external/usr/` whereas the `--root` option installs
7983
# everything under `external/`. That's why we include both in the PYTHONPATH
8084

81-
export PATH=$(pwd)/external/usr/bin:$PATH
8285
export PYTHONPATH=$(pwd)/external:$(pwd)/external/usr/lib/python$pyver/site-packages:$PYTHONPATH
8386

8487
CMD $python -m pip install --no-cache-dir -q --upgrade pip --target=external/

config/cscs.py

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -826,7 +826,7 @@
826826
{
827827
'name': 'PrgEnv-pgi-nompi-nocuda',
828828
'target_systems': [
829-
'arolla', 'tsa'
829+
'arolla',
830830
],
831831
'modules': [
832832
'PrgEnv-pgi/19.9-nocuda'
@@ -835,10 +835,22 @@
835835
'cxx': 'pgc++',
836836
'ftn': 'pgf90'
837837
},
838+
{
839+
'name': 'PrgEnv-pgi-nompi-nocuda',
840+
'target_systems': [
841+
'tsa',
842+
],
843+
'modules': [
844+
'PrgEnv-pgi/20.4-nocuda'
845+
],
846+
'cc': 'pgcc',
847+
'cxx': 'pgc++',
848+
'ftn': 'pgf90'
849+
},
838850
{
839851
'name': 'PrgEnv-pgi-nompi',
840852
'target_systems': [
841-
'arolla', 'tsa'
853+
'arolla',
842854
],
843855
'modules': [
844856
'PrgEnv-pgi/19.9'
@@ -847,10 +859,22 @@
847859
'cxx': 'pgc++',
848860
'ftn': 'pgf90'
849861
},
862+
{
863+
'name': 'PrgEnv-pgi-nompi',
864+
'target_systems': [
865+
'tsa',
866+
],
867+
'modules': [
868+
'PrgEnv-pgi/20.4'
869+
],
870+
'cc': 'pgcc',
871+
'cxx': 'pgc++',
872+
'ftn': 'pgf90'
873+
},
850874
{
851875
'name': 'PrgEnv-pgi',
852876
'target_systems': [
853-
'arolla', 'tsa'
877+
'arolla',
854878
],
855879
'modules': [
856880
'PrgEnv-pgi/19.9'
@@ -859,10 +883,22 @@
859883
'cxx': 'mpicxx',
860884
'ftn': 'mpifort'
861885
},
886+
{
887+
'name': 'PrgEnv-pgi',
888+
'target_systems': [
889+
'tsa',
890+
],
891+
'modules': [
892+
'PrgEnv-pgi/20.4'
893+
],
894+
'cc': 'mpicc',
895+
'cxx': 'mpicxx',
896+
'ftn': 'mpifort'
897+
},
862898
{
863899
'name': 'PrgEnv-pgi-nocuda',
864900
'target_systems': [
865-
'arolla', 'tsa'
901+
'arolla',
866902
],
867903
'modules': [
868904
'PrgEnv-pgi/19.9-nocuda'
@@ -871,6 +907,18 @@
871907
'cxx': 'mpicxx',
872908
'ftn': 'mpifort'
873909
},
910+
{
911+
'name': 'PrgEnv-pgi-nocuda',
912+
'target_systems': [
913+
'tsa',
914+
],
915+
'modules': [
916+
'PrgEnv-pgi/20.4-nocuda'
917+
],
918+
'cc': 'mpicc',
919+
'cxx': 'mpicxx',
920+
'ftn': 'mpifort'
921+
},
874922
{
875923
'name': 'PrgEnv-gnu',
876924
'target_systems': [

cscs-checks/prgenv/cuda-fortran/cuda_fortran_check.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,7 @@ def __init__(self):
1313
self.valid_systems = ['daint:gpu', 'dom:gpu']
1414
self.valid_prog_environs = ['PrgEnv-pgi']
1515
self.sourcepath = 'vecAdd_cuda.cuf'
16-
17-
# FIXME: PGI 20.x does not support CUDA 11, see case #275674
18-
self.modules = ['craype-accel-nvidia60',
19-
'cudatoolkit/10.2.89_3.29-7.0.2.1_3.5__g67354b4']
16+
self.modules = ['craype-accel-nvidia60']
2017
self.build_system = 'SingleSource'
2118
self.build_system.fflags = ['-ta=tesla:cc60']
2219
self.num_gpus_per_node = 1
@@ -25,3 +22,13 @@ def __init__(self):
2522
self.sanity_patterns = sn.assert_reference(result, 1., -1e-5, 1e-5)
2623
self.maintainers = ['TM', 'AJ']
2724
self.tags = {'production', 'craype'}
25+
26+
# FIXME: PGI 20.x does not support CUDA 11, see case #275674
27+
@rfm.run_before('compile')
28+
def cudatoolkit_pgi_20x_workaround(self):
29+
if self.current_system.name == 'daint':
30+
cudatoolkit_version = '10.2.89_3.29-7.0.2.1_3.5__g67354b4'
31+
else:
32+
cudatoolkit_version = '10.2.89_3.29-7.0.2.1_3.27__g67354b4'
33+
34+
self.modules += [f'cudatoolkit/{cudatoolkit_version}']

docs/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
jsonschema==3.2.0
22
semver==2.13.0
3-
Sphinx==3.5.2
3+
Sphinx==3.5.3
44
sphinx-rtd-theme==0.5.1

docs/tutorial_advanced.rst

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -790,3 +790,63 @@ Therefore, the ``release.txt`` file can now be used in the subsequent sanity che
790790
791791
For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide.
792792
On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`.
793+
794+
795+
Writing reusable tests
796+
----------------------
797+
798+
.. versionadded:: 3.5.0
799+
800+
So far, all the examples shown above were tight to a particular system or configuration, which makes reusing these tests in other systems not straightforward.
801+
However, the introduction of the :py:func:`~reframe.core.pipeline.RegressionTest.parameter` and :py:func:`~reframe.core.pipeline.RegressionTest.variable` ReFrame built-ins solves this problem, eliminating the need to specify any of the test variables in the :func:`__init__` method.
802+
Hence, these parameters and variables can be treated as simple class attributes, which allows us to leverage Python's class inheritance and write more modular tests.
803+
For simplicity, we illustrate this concept with the above :class:`ContainerTest` example, where the goal here is to re-write this test as a library that users can simply import from and derive their tests without having to rewrite the bulk of the test.
804+
Also, for illustrative purposes, we parameterize this library test on a few different image tags (the above example just used ``ubuntu:18.04``) and throw the container commands into a separate bash script just to create some source files.
805+
Thus, removing all the system and configuration specific variables, and moving as many assignments as possible into the class body, the system agnostic library test looks as follows:
806+
807+
.. code-block:: console
808+
809+
cat tutorials/advanced/library/lib/__init__.py
810+
811+
812+
.. literalinclude:: ../tutorials/advanced/library/lib/__init__.py
813+
:lines: 6-
814+
:emphasize-lines: 8-17
815+
816+
Note that the class :class:`ContainerBase` is not decorated since it does not specify the required variables ``valid_systems`` and ``valid_prog_environs``, and it declares the ``platform`` parameter without any defined values assigned.
817+
Hence, the user can simply derive from this test and specialize it to use the desired container platforms.
818+
Since the parameters are defined directly in the class body, the user is also free to override or extend any of the other parameters in a derived test.
819+
In this example, we have parametrized the base test to run with the ``ubuntu:18.04`` and ``ubuntu:20.04`` images, but these values from ``dist`` (and also the ``dist_name`` variable) could be modified by the derived class if needed.
820+
821+
On the other hand, the rest of the test depends on the values from the test parameters, and a parameter is only assigned a specific value after the class has been instantiated.
822+
Thus, the rest of the test is expressed as hooks, without the need to write anything in the :func:`__init__` method.
823+
In fact, writing the test in this way permits having hooks that depend on undefined variables or parameters.
824+
This is the case with the :func:`set_container_platform` hook, which depends on the undefined parameter ``platform``.
825+
Hence, the derived test **must** define all the required parameters and variables; otherwise ReFrame will notice that the test is not well defined and will raise an error accordingly.
826+
827+
Before moving onwards to the derived test, note that the :class:`ContainerBase` class takes the additional argument ``pin_prefix=True``, which locks the prefix of all derived tests to this base test.
828+
This will allow the retrieval of the sources located in the library by any derived test, regardless of what their containing directory is.
829+
830+
.. code-block:: console
831+
832+
cat tutorials/advanced/library/lib/src/get_os_release.sh
833+
834+
835+
.. literalinclude:: ../tutorials/advanced/library/lib/src/get_os_release.sh
836+
:lines: 1-
837+
838+
Now from the user's perspective, the only thing to do is to import the above base test and specify the required variables and parameters.
839+
For consistency with the above example, we set the ``platform`` parameter to use Sarus and Singularity, and we configure the test to run on Piz Daint with the built-in programming environment.
840+
Hence, the above :class:`ContainerTest` is now reduced to the following:
841+
842+
.. code-block:: console
843+
844+
cat tutorials/advanced/library/usr/container_test.py
845+
846+
847+
.. literalinclude:: ../tutorials/advanced/library/usr/container_test.py
848+
:lines: 6-
849+
850+
In a similar fashion, any other user could reuse the above :class:`ContainerBase` class and write the test for their own system with a few lines of code.
851+
852+
*Happy test sharing!*

reframe/core/buildsystems.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,11 @@ def post_build(self, buildjob):
147147
Build systems may use this information to do some post processing and
148148
provide additional, build system-specific, functionality to the users.
149149
150+
This function will always be executed from the test's stage directory.
151+
150152
.. versionadded:: 3.5.0
153+
.. versionchanged:: 3.5.2
154+
The function is executed from the stage directory.
151155
152156
:meta private:
153157
@@ -741,7 +745,6 @@ def __init__(self):
741745
self.package_opts = {}
742746
self.prefix = 'easybuild'
743747
self._eb_modules = []
744-
self._prefix_save = None
745748

746749
def emit_build_commands(self, environ):
747750
if not self.easyconfigs:
@@ -755,7 +758,6 @@ def emit_build_commands(self, environ):
755758

756759
prefix = os.path.join(os.getcwd(), self.prefix)
757760
options = ' '.join(self.options)
758-
self._prefix_save = prefix
759761
return [f'export EASYBUILD_BUILDPATH={prefix}/build',
760762
f'export EASYBUILD_INSTALLPATH={prefix}',
761763
f'export EASYBUILD_PREFIX={prefix}',
@@ -765,7 +767,8 @@ def emit_build_commands(self, environ):
765767
def post_build(self, buildjob):
766768
# Store the modules generated by EasyBuild
767769

768-
modulesdir = os.path.join(self._prefix_save, 'modules', 'all')
770+
modulesdir = os.path.join(os.getcwd(), self.prefix,
771+
'modules', 'all')
769772
with open(buildjob.stdout) as fp:
770773
out = fp.read()
771774

0 commit comments

Comments
 (0)