Merge branch 'master' into bugfix/parent_class_collisions

Author: vkarak

ci-scripts/genrelnotes.py

@@ -5,7 +5,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import collections
 import re
 import sys
 import subprocess
@@ -15,8 +14,8 @@ def usage():
     sys.stderr.write('Usage: %s PREV_RELEASE CURR_RELEASE\n' % sys.argv[0])
 
 
-def extract_release_notes(git_output, tag):
-    return re.findall(r'pull request (#\d+).*\s*\[%s\] (.*)' % tag, git_output)
+def extract_release_notes(git_output):
+    return re.findall(r'pull request (#\d+).*\s*\[(\S+)\] (.*)', git_output)
 
 
 if __name__ == '__main__':
@@ -39,21 +38,30 @@ def extract_release_notes(git_output, tag):
         sys.stderr.write(e.stdout)
         sys.exit(1)
 
-    titles = {
-        'feat': '## New features and enhancements',
+    tag_mapping = {
+        'feat': '## New features',
+        'enhancement': '## Enhancements',
+        'doc': '## Enhancements',
         'bugfix': '## Bug fixes',
         'testlib': '## Test library'
     }
-    sections = collections.OrderedDict()
-    for tag in ['feat', 'bugfix', 'testlib', 'ci', 'doc']:
-        title_line = titles.get(tag, '## Other')
-        sections.setdefault(title_line, [])
-        for pr, descr in extract_release_notes(completed.stdout, tag):
-            descr_line = '- %s (%s)' % (descr, pr)
-            sections[title_line].append(descr_line)
+    sections = {
+        '## New features': [],
+        '## Enhancements': [],
+        '## Bug fixes': [],
+        '## Test library': [],
+        '## Other': []
+    }
+    for pr, tag, descr in extract_release_notes(completed.stdout):
+        title_line = tag_mapping.get(tag, '## Other')
+        descr_line = f'- {descr} ({pr})'
+        sections[title_line].append(descr_line)
 
     print('# Release Notes')
     for sec_title, sec_lines in sections.items():
+        if not sec_lines:
+            continue
+
         print()
         print(sec_title)
         print()

docs/conf.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
 # ReFrame Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: BSD-3-Clause
@@ -85,7 +85,7 @@
 
 # General information about the project.
 project = 'ReFrame'
-copyright = '2016-2022, CSCS/ETH Zurich and other ReFrame Project Developers'
+copyright = '2016-2023, CSCS/ETH Zurich, ReFrame Project Developers'
 author = 'ReFrame Project Developers'
 
 # The version info for the project you're documenting, acts as replacement for

docs/deferrable_functions_reference.rst

@@ -58,7 +58,7 @@ Currently ReFrame provides three broad categories of deferrable functions:
    They include, but are not limited to, functions to iterate over regex matches in a file, extracting and converting values from regex matches, computing statistical information on series of data etc.
 
 
- .. _deferrable-performance-functions:
+.. _deferrable-performance-functions:
 
 
 --------------------------------

docs/manpage.rst

@@ -220,6 +220,10 @@ An action must always be specified.
    .. versionadded:: 3.10.0
       Support for different types of listing is added.
 
+   .. versionchanged:: 4.0.5
+      The variable names to which fixtures are bound are also listed.
+      See :ref:`test_naming_scheme` for more information.
+
 .. option:: -l, --list[=T|C]
 
    List selected tests and their dependencies.
@@ -234,6 +238,10 @@ An action must always be specified.
    .. versionadded:: 3.10.0
       Support for different types of listing is added.
 
+   .. versionchanged:: 4.0.5
+      The variable names to which fixtures are bound are also listed.
+      See :ref:`test_naming_scheme` for more information.
+
 .. option:: --list-tags
 
    List the unique tags of the selected tests.
@@ -510,7 +518,8 @@ Options controlling ReFrame execution
 
    ``TEST`` can have the form ``[TEST.][FIXT.]*``, in which case ``VAR`` will be set in fixture ``FIXT`` of ``TEST``.
    Note that this syntax is recursive on fixtures, so that a variable can be set in a fixture arbitrarily deep.
-   ``TEST`` prefix refers to the test class name, *not* the test name, but ``FIXT`` refers to the fixture name *inside* the referenced test.
+   ``TEST`` prefix refers to the test class name, *not* the test name and ``FIXT`` refers to the fixture *variable name* inside the referenced test, i.e., the test variable to which the fixture is bound.
+   The fixture variable name is referred to as ``'<varname>`` when listing tests with the :option:`-l` and :option:`-L` options.
 
    Multiple variables can be set at the same time by passing this option multiple times.
    This option *cannot* change arbitrary test attributes, but only test variables declared with the :attr:`~reframe.core.pipeline.RegressionMixin.variable` built-in.
@@ -951,24 +960,29 @@ Here is how this test is listed where the various components of the display name
 
 .. code-block:: console
 
-   - TestA %x=4 %l.foo=10 %t.p=2 /1c51609b
-       ^Myfixture %p=1 ~TestA_3 /f027ee75
-       ^MyFixture %p=2 ~TestA_3 /830323a4
-       ^X %foo=10 ~generic:default+builtin /7dae3cc5
-   - TestA %x=3 %l.foo=10 %t.p=2 /707b752c
-       ^MyFixture %p=1 ~TestA_2 /02368516
-       ^MyFixture %p=2 ~TestA_2 /854b99b5
-       ^X %foo=10 ~generic:default+builtin /7dae3cc5
-   - TestA %x=4 %l.foo=10 %t.p=1 /c65657d5
-       ^MyFixture %p=2 ~TestA_1 /f0383f7f
-       ^MyFixture %p=1 ~TestA_1 /d07f4281
-       ^X %foo=10 ~generic:default+builtin /7dae3cc5
-   - TestA %x=3 %l.foo=10 %t.p=1 /1b9f44df
-       ^MyFixture %p=2 ~TestA_0 /b894ab05
-       ^MyFixture %p=1 ~TestA_0 /ca376ca8
-       ^X %foo=10 ~generic:default+builtin /7dae3cc5
+   - TestA %x=4 %l.foo=10 %t.p=2 /8804be5d
+       ^MyFixture %p=1 ~TestA_3 't 'f /f027ee75
+       ^MyFixture %p=2 ~TestA_3 't 'f /830323a4
+       ^X %foo=10 ~generic:default+builtin 'l /7dae3cc5
+   - TestA %x=3 %l.foo=10 %t.p=2 /89f6f5d1
+       ^MyFixture %p=1 ~TestA_2 't 'f /02368516
+       ^MyFixture %p=2 ~TestA_2 't 'f /854b99b5
+       ^X %foo=10 ~generic:default+builtin 'l /7dae3cc5
+   - TestA %x=4 %l.foo=10 %t.p=1 /af9b2941
+       ^MyFixture %p=2 ~TestA_1 't 'f /f0383f7f
+       ^MyFixture %p=1 ~TestA_1 't 'f /d07f4281
+       ^X %foo=10 ~generic:default+builtin 'l /7dae3cc5
+   - TestA %x=3 %l.foo=10 %t.p=1 /a9e50aa3
+       ^MyFixture %p=2 ~TestA_0 't 'f /b894ab05
+       ^MyFixture %p=1 ~TestA_0 't 'f /ca376ca8
+       ^X %foo=10 ~generic:default+builtin 'l /7dae3cc5
    Found 4 check(s)
 
+Notice that the variable name to which every fixture is bound in its parent test is also listed as ``'<varname>``.
+This is useful for setting variables down the fixture hierarchy using the :option:`-S` option.
+
+
+
 Display names may not always be unique.
 Assume the following test:
 

docs/tutorial_basics.rst

@@ -429,12 +429,13 @@ For running the benchmark, we need to set the OpenMP number of threads and pin t
 You can set environment variables in a ReFrame test through the :attr:`~reframe.core.pipeline.RegressionTest.env_vars` dictionary.
 
 What makes a ReFrame test a performance test is the definition of at least one :ref:`performance function<deferrable-performance-functions>`.
-Similarly to a test's :func:`@sanity_function<reframe.core.pipeline.RegressionMixin.sanity_function>`, a performance function is a member function decorated with the :attr:`@performance_function<reframe.core.pipeline.RegressionMixin.performance_function>` decorator, which binds the decorated function to a given unit.
-These functions can be used by the regression test to extract, measure or compute a given quantity of interest; where in this context, the values returned by a performance function are referred to as performance variables.
-Alternatively, performance functions can also be thought as `tools` available to the regression test for extracting performance variables.
-By default, ReFrame will attempt to execute all the available performance functions during the test's ``performance`` stage, producing a single performance variable out of each of the available performance functions.
-These default-generated performance variables are defined in the regression test's attribute :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` during class instantiation, and their default name matches the name of their associated performance function.
-However, one could customize the default-generated performance variable's name by passing the ``perf-key`` argument to the :attr:`@performance_function<reframe.core.pipeline.RegressionMixin.performance_function>` decorator of the associated performance function.
+Similarly to a test's :func:`@sanity_function<reframe.core.builtins.sanity_function>`, a performance function is a member function decorated with the :func:`@performance_function<reframe.core.builtins.performance_function>` decorator that merely extracts or computes a performance metric from the test's output and associates it with a unit.
+By default, every performance function defined in the test is assigned to a *performance variable* with the function's name.
+A performance variable is a named quantity representing a performance metric that ReFrame will report on, log and can also check against a reference value.
+The performance variables of a test are stored in the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary.
+The keys are the names of the metrics, whereas the values are :ref:`performance functions <deferrable-performance-functions>`.
+The :func:`@performance_function<reframe.core.builtins.performance_function>` decorator apart from turning an ordinary method into a "performance function", it also creates an entry in the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary.
+The optional ``perf_key`` argument can be used to assign a different name to the newly created performance variable.
 
 In this example, we extract four performance variables, namely the memory bandwidth values for each of the "Copy", "Scale", "Add" and "Triad" sub-benchmarks of STREAM, where each of the performance functions use the :func:`~reframe.utility.sanity.extractsingle` utility function.
 For each of the sub-benchmarks we extract the "Best Rate MB/s" column of the output (see below) and we convert that to a float.
@@ -468,9 +469,11 @@ This is especially useful if you run long suites of performance exploration test
 Setting explicitly the test's performance variables
 ---------------------------------------------------
 
-In the above STREAM example, all four performance functions were almost identical except for a small part of the regex pattern, which led to some code repetition.
-Even though the performance functions were rather simple and the code repetition was not much in that case, this is still not a good practice and it is certainly an approach that would not scale when using more complex performance functions.
-Hence, in this example, we show how to collapse all these four performance functions into a single function and how to reuse this single performance function to create multiple performance variables.
+Users are allowed to manipulate the test's :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary directly.
+This is useful to avoid code repetition or in cases that relying on decorated methods to populate the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` is impractical, e.g., creating multiple performance variables in a loop.
+
+You might have noticed that in our STREAM example above, all four performance functions are almost identical except for a small part of the regex pattern.
+In the following example, we define a single performance function, :func:`extract_bw`, that can extract any of the requested bandwidth metrics, and we populate the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` ourselves in a pre-performance hook:
 
 .. code-block:: console
 
@@ -480,28 +483,29 @@ Hence, in this example, we show how to collapse all these four performance funct
    :start-at: import reframe
    :emphasize-lines: 28-
 
-As shown in the highlighted lines, this example collapses the four performance functions from the previous example into the :func:`extract_bw` function, which is also decorated with the :attr:`@performance_function<reframe.core.pipeline.RegressionMixin.performance_function>` decorator with the units set to ``'MB/s'``.
-However, the :func:`extract_bw` function now takes the optional argument ``kind`` which selects the STREAM benchmark to extract.
-By default, this argument is set to ``'Copy'`` because functions decorated with :attr:`@performance_function<reframe.core.pipeline.RegressionMixin.performance_function>` are only allowed to have ``self`` as a non-default argument.
-Thus, from this performance function definition, ReFrame will default-generate a single performance variable during the test instantiation under the name ``extract_bw``, where this variable will report the performance results from the ``Copy`` benchmark.
-With no further action from our side, ReFrame would just report the performance of the test based on this default-generated performance variable, but that is not what we are after here.
-Therefore, we must modify these default performance variables so that this version of the STREAM test produces the same results as in the previous example.
-As mentioned before, the performance variables (also the default-generated ones) are stored in the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary, so all we need to do is to redefine this mapping with our desired performance variables as done in the pre-performance pipeline hook :func:`set_perf_variables`.
+As mentioned in the previous section the :func:`@performance_function <reframe.core.builtins.performance_function>` decorator performs two tasks:
 
-.. tip::
-   Performance functions may also be generated inline using the :func:`~reframe.utility.sanity.make_performance_function` utility as shown below.
-
-   .. code-block:: python
-
-      @run_before('performance')
-      def set_perf_vars(self):
-          self.perf_variables = {
-              'Copy': sn.make_performance_function(
-                  sn.extractsingle(r'Copy:\s+(\S+)\s+.*',
-                                   self.stdout, 1, float),
-                  'MB/s'
-               )
-          }
+1. It converts a test method to *performance function*, i.e., a function that is suitable for extracting a performance metric.
+2. It updates the :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` dictionary with the newly created performance function.
+
+In this example, we are only interested in the first functionality and that's why we redefine completely the test's :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` using the :func:`extract_bw` performance function.
+If you are inheriting from a base test and you don't want to override completely its performance variables, you could call instead :py:func:`update` on :attr:`~reframe.core.pipeline.RegressionTest.perf_variables`.
+
+Finally, you can convert any arbitrary function or :doc:`deferred expression <deferrable_functions_reference>` into a performance function by calling the :func:`~reframe.utility.sanity.make_performance_function` utility as shown below:
+
+.. code-block:: python
+
+   @run_before('performance')
+   def set_perf_vars(self):
+       self.perf_variables = {
+           'Copy': sn.make_performance_function(
+               sn.extractsingle(r'Copy:\s+(\S+)\s+.*',
+                                self.stdout, 1, float),
+               'MB/s'
+            )
+       }
+
+Note that in this case, the newly created performance function is not assigned to a test's performance variable and you will have to do this independently.
 
 -----------------------
 Adding reference values

reframe/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
 # ReFrame Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: BSD-3-Clause

reframe/core/backends.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
 # ReFrame Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: BSD-3-Clause

reframe/core/buildsystems.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
 # ReFrame Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: BSD-3-Clause

reframe/core/builtins.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
 # ReFrame Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: BSD-3-Clause

reframe/core/config.py

@@ -1,4 +1,4 @@
-# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# Copyright 2016-2023 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
 # ReFrame Project Developers. See the top-level LICENSE file for details.
 #
 # SPDX-License-Identifier: BSD-3-Clause