Merge branch 'master' into ci-cli-options

Author: Vasileios Karakasis

cscs-checks/system/jobreport/gpu_report.py

@@ -8,11 +8,11 @@
 import time
 
 from reframe.core.exceptions import SanityError
-from hpctestlib.microbenchmarks.gpu.gpu_burn import GpuBurn
+from hpctestlib.microbenchmarks.gpu.gpu_burn import gpu_burn_check
 
 
 @rfm.simple_test
-class gpu_usage_report_check(GpuBurn):
+class gpu_usage_report_check(gpu_burn_check):
     '''Check the output from the job report.
 
     This check uses the gpu burn app and checks that the job report produces
@@ -23,14 +23,10 @@ class gpu_usage_report_check(GpuBurn):
     '''
 
     valid_systems = ['daint:gpu', 'dom:gpu']
-    valid_prog_environs = ['PrgEnv-gnu']
     descr = 'Check GPU usage from job report'
     gpu_build = 'cuda'
-    modules = ['craype-accel-nvidia60', 'cdt-cuda']
-    num_tasks = 0
+    num_tasks = 2
     num_gpus_per_node = 1
-    burn_time = variable(int, value=10)
-    executable_opts = ['-d', f'{burn_time}']
     perf_floor = variable(float, value=-0.2)
     tags = {'production'}
 
@@ -44,7 +40,7 @@ def set_launcher_opts(self):
         self.job.launcher.options = ['-u']
 
     @sanity_function
-    def set_sanity_patterns(self):
+    def assert_jobreport_success(self):
         '''Extend sanity and wait for the jobreport.
 
         If a large number of nodes is used, the final jobreport output happens
@@ -59,57 +55,35 @@ def set_sanity_patterns(self):
         except SanityError:
             time.sleep(25)
 
-        return sn.all([
-            self.count_successful_burns(), self.gpu_usage_sanity()
-        ])
+        return self.assert_successful_burn_count(), self.gpu_usage_sanity()
 
     @deferrable
     def gpu_usage_sanity(self):
         '''Verify that the jobreport output has sensible numbers.
 
-        This function asserts that the nodes reported are at least a subset of
-        all nodes used by the gpu burn app. Also, the GPU usage is verified by
-        assuming that in the worst case scenario, the usage is near 100% during
-        the burn, and 0% outside the burn period. Lastly, the GPU usage time
-        for each node is also asserted to be greater or equal than the burn
-        time.
+        The GPU usage is verified by assuming that in the worst case scenario,
+        the usage is near 100% during the burn, and 0% outside the burn period.
+        Lastly, the GPU usage time for each node is also asserted to be greater
+        or equal than the burn time.
         '''
 
-        # Get set with all nodes
-        patt = r'^\s*\[([^\]]*)\]\s*GPU\s*\d+\(OK\)'
-        full_node_set = set(sn.extractall(patt, self.stdout, 1))
-
         # Parse job report data
         patt = r'^\s*(\w*)\s*(\d+)\s*%\s*\d+\s*MiB\s*\d+:\d+:(\d+)'
         self.nodes_reported = sn.extractall(patt, self.stdout, 1)
+        self.num_tasks_assigned = self.num_tasks * self.num_gpus_per_node
         usage = sn.extractall(patt, self.stdout, 2, int)
         time_reported = sn.extractall(patt, self.stdout, 3, int)
         return sn.all([
             sn.assert_ge(sn.count(self.nodes_reported), 1),
-            set(self.nodes_reported).issubset(full_node_set),
             sn.all(
-                map(lambda x, y: self.burn_time/x <= y, time_reported, usage)
+                map(lambda x, y: self.duration/x <= y/100, time_reported, usage)
             ),
-            sn.assert_ge(sn.min(time_reported), self.burn_time)
+            sn.assert_ge(sn.min(time_reported), self.duration)
         ])
 
-    @performance_function('nodes')
-    def total_nodes_reported(self):
-        return sn.count(self.nodes_reported)
-
-    @run_before('performance')
-    def set_perf_variables(self):
-        '''The number of reported nodes can be used as a perf metric.
-
-        For now, the low limit can go to zero, but this can be set to a more
-        restrictive value.
-        '''
-
-        self.reference = {
-            '*': {
-                'nodes_reported': (self.num_tasks, self.perf_floor, 0)
-            },
-        }
-        self.perf_variables = {
-            'nodes_reported': self.total_nodes_reported()
-        }
+    @deferrable
+    def assert_successful_burn_count(self):
+        '''Assert that the expected successful burn count is reported.'''
+        return sn.assert_eq(sn.count(sn.findall(r'^GPU\s*\d+\(OK\)',
+                                                self.stdout)),
+                            self.num_tasks_assigned)

docs/manpage.rst

@@ -484,6 +484,10 @@ Options controlling ReFrame execution
 
    Set variable ``VAR`` in all tests or optionally only in test ``TEST`` to ``VAL``.
 
+   ``TEST`` can have the form ``[TEST.][FIXT.]*``, in which case ``VAR`` will be set in fixture ``FIXT`` of ``TEST``.
+   Note that this syntax is recursive on fixtures, so that a variable can be set in a fixture arbitrarily deep.
+   ``TEST`` prefix refers to the test class name, *not* the test name, but ``FIXT`` refers to the fixture name *inside* the referenced test.
+
    Multiple variables can be set at the same time by passing this option multiple times.
    This option *cannot* change arbitrary test attributes, but only test variables declared with the :attr:`~reframe.core.pipeline.RegressionMixin.variable` built-in.
    If an attempt is made to change an inexistent variable or a test parameter, a warning will be issued.
@@ -511,8 +515,6 @@ Options controlling ReFrame execution
    Conversions to arbitrary objects are also supported.
    See :class:`~reframe.utility.typecheck.ConvertibleType` for more details.
 
-   The optional ``TEST.`` prefix refers to the test class name, *not* the test name.
-
    Variable assignments passed from the command line happen *before* the test is instantiated and is the exact equivalent of assigning a new value to the variable *at the end* of the test class body.
    This has a number of implications that users of this feature should be aware of:
 
@@ -561,6 +563,10 @@ Options controlling ReFrame execution
 
       Proper handling of boolean variables.
 
+   .. versionchanged:: 3.11.1
+
+      Allow setting variables in fixtures.
+
 
 .. option:: --skip-performance-check
 

reframe/core/meta.py

@@ -535,6 +535,17 @@ def setvar(cls, name, value):
 
         '''
 
+        if '.' in name:
+            # `name` refers to a fixture variable
+            fixtname, varname = name.split('.', maxsplit=1)
+            try:
+                fixt_space = super().__getattribute__('_rfm_fixture_space')
+            except AttributeError:
+                '''Catch early access attempt to the variable space.'''
+
+            if fixtname in fixt_space:
+                return fixt_space[fixtname].cls.setvar(varname, value)
+
         try:
             var_space = super().__getattribute__('_rfm_var_space')
             if name in var_space:

reframe/core/schedulers/pbs.py

@@ -181,6 +181,13 @@ def _update_nodelist(self, job, nodespec):
         job._nodelist.sort()
 
     def poll(self, *jobs):
+        def output_ready(job):
+            # We report a job as finished only when its stdout/stderr are
+            # written back to the working directory
+            stdout = os.path.join(job.workdir, job.stdout)
+            stderr = os.path.join(job.workdir, job.stderr)
+            return os.path.exists(stdout) and os.path.exists(stderr)
+
         if jobs:
             # Filter out non-jobs
             jobs = [job for job in jobs if job is not None]
@@ -198,11 +205,12 @@ def poll(self, *jobs):
         # Otherwise, it will return with return code 0 and print information
         # only for the jobs it could find.
         if completed.returncode in (153, 35):
-            self.log(f'Return code is {completed.returncode}: '
-                     f'assuming all jobs completed')
+            self.log(f'Return code is {completed.returncode}')
             for job in jobs:
                 job._state = 'COMPLETED'
-                job._completed = True
+                if job.cancelled or output_ready(job):
+                    self.log(f'Assuming job {job.jobid} completed')
+                    job._completed = True
 
             return
 
@@ -224,10 +232,12 @@ def poll(self, *jobs):
 
         for job in jobs:
             if job.jobid not in jobinfo:
-                self.log(f'Job {job.jobid} not known to scheduler, '
-                         f'assuming job completed')
+                self.log(f'Job {job.jobid} not known to scheduler')
                 job._state = 'COMPLETED'
-                job._completed = True
+                if job.cancelled or output_ready(job):
+                    self.log(f'Assuming job {job.jobid} completed')
+                    job._completed = True
+
                 continue
 
             info = jobinfo[job.jobid]
@@ -259,10 +269,7 @@ def poll(self, *jobs):
 
                 # We report a job as finished only when its stdout/stderr are
                 # written back to the working directory
-                stdout = os.path.join(job.workdir, job.stdout)
-                stderr = os.path.join(job.workdir, job.stderr)
-                out_ready = os.path.exists(stdout) and os.path.exists(stderr)
-                done = job.cancelled or out_ready
+                done = job.cancelled or output_ready(job)
                 if done:
                     job._completed = True
             elif (job.state in ['QUEUED', 'HELD', 'WAITING'] and

reframe/core/variables.py

@@ -281,7 +281,7 @@ def __getattr__(self, name):
     def _check_is_defined(self):
         if not self.is_defined():
             raise ReframeSyntaxError(
-                f'variable {self._name} is not assigned a value'
+                f'variable {self._name!r} is not assigned a value'
             )
 
     def __repr__(self):

unittests/resources/checks_unlisted/externalvars.py

@@ -3,19 +3,33 @@
 import reframe.utility.typecheck as typ
 
 
+class Bacon(rfm.RunOnlyRegressionTest):
+    bacon = variable(int, value=-1)
+    executable = 'echo'
+    sanity_patterns = sn.assert_true(1)
+
+
+class Eggs(rfm.RunOnlyRegressionTest):
+    eggs = fixture(Bacon)
+    executable = 'echo'
+    sanity_patterns = sn.assert_true(1)
+
+
 @rfm.simple_test
 class external_x(rfm.RunOnlyRegressionTest):
     valid_systems = ['*']
     valid_prog_environs = ['*']
     foo = variable(int, value=1)
     ham = variable(typ.Bool, value=False)
+    spam = fixture(Eggs)
     executable = 'echo'
 
     @sanity_function
     def assert_foo(self):
         return sn.all([
             sn.assert_eq(self.foo, 3),
-            sn.assert_true(self.ham)
+            sn.assert_true(self.ham),
+            sn.assert_eq(self.spam.eggs.bacon, 10)
         ])
 
 

unittests/test_cli.py

@@ -837,13 +837,17 @@ def test_detect_host_topology_file(run_reframe, tmp_path):
 def test_external_vars(run_reframe):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/externalvars.py'],
-        more_options=['-S', 'external_x.foo=3', '-S', 'external_y.foo=2',
-                      '-S', 'foolist=3,4', '-S', 'bar=@none',
+        more_options=['-S', 'external_x.foo=3',
                       '-S', 'external_x.ham=true',
-                      '-S', 'external_y.baz=false']
+                      '-S', 'external_x.spam.eggs.bacon=10',
+                      '-S', 'external_y.foo=2',
+                      '-S', 'external_y.baz=false',
+                      '-S', 'foolist=3,4',
+                      '-S', 'bar=@none']
     )
+    assert 'PASSED' in stdout
+    assert 'Ran 6/6 test case(s)' in stdout
     assert 'Traceback' not in stdout
-    assert 'Ran 2/2 test case(s)' in stdout
     assert 'Traceback' not in stderr
     assert returncode == 0