Merge pull request #1183 from victorusu/framework/perf-value-str-fail-nicely

[bugfix] Fix TypeError crash in the performance checking phase when the extracted performance variable value is not a number

Author: Vasileios Karakasis

reframe/core/pipeline.py

@@ -15,6 +15,7 @@
 import functools
 import inspect
 import itertools
+import numbers
 import os
 import shutil
 
@@ -37,7 +38,6 @@
 from reframe.core.schedulers import Job
 from reframe.core.schedulers.registry import getscheduler
 from reframe.core.systems import SystemPartition
-from reframe.utility.sanity import assert_reference
 
 
 # Dependency kinds
@@ -1321,10 +1321,18 @@ def check_performance(self):
 
             for key, values in self._perfvalues.items():
                 val, ref, low_thres, high_thres, *_ = values
+
+                # Verify that val is a number
+                if not isinstance(val, numbers.Number):
+                    raise SanityError(
+                        "the value extracted for performance variable '%s' "
+                        "is not a number: %s" % (key, val)
+                    )
+
                 tag = key.split(':')[-1]
                 try:
                     sn.evaluate(
-                        assert_reference(
+                        sn.assert_reference(
                             val, ref, low_thres, high_thres,
                             msg=('failed to meet reference: %s={0}, '
                                  'expected {1} (l={2}, u={3})' % tag))

reframe/utility/sanity.py

@@ -63,6 +63,7 @@
    computing statistical information on series of data etc.
 
 '''
+
 import builtins
 import glob as pyglob
 import itertools
@@ -551,7 +552,7 @@ def calc_bound(thres):
         evaluate(assert_bounded(val, lower, upper))
     except SanityError:
         error_msg = msg or '{0} is beyond reference value {1} (l={2}, u={3})'
-        raise SanityError(_format(error_msg, val, ref, lower, upper))
+        raise SanityError(_format(error_msg, val, ref, lower, upper)) from None
     else:
         return True
 

unittests/test_pipeline.py

@@ -740,7 +740,6 @@ def setUp(self):
             'value3': sn.extractsingle(r'performance3 = (\S+)',
                                        self.perf_file.name, 1, float)
         }
-
         self.test.sanity_patterns = sn.assert_found(r'result = success',
                                                     self.output_file.name)
 
@@ -896,6 +895,21 @@ def test_tag_resolution(self):
         }
         self.test.check_performance()
 
+    def test_invalid_perf_value(self):
+        self.test.perf_patterns = {
+            'value1': sn.extractsingle(r'performance1 = (\S+)',
+                                       self.perf_file.name, 1, float),
+            'value2': sn.extractsingle(r'performance2 = (\S+)',
+                                       self.perf_file.name, 1, str),
+            'value3': sn.extractsingle(r'performance3 = (\S+)',
+                                       self.perf_file.name, 1, float)
+        }
+        self.write_performance_output(performance1=1.3,
+                                      performance2='foo',
+                                      performance3=3.3)
+        with pytest.raises(SanityError, match='not a number'):
+            self.test.check_performance()
+
     def test_perf_var_evaluation(self):
         # All performance values must be evaluated, despite the first one
         # failing To test this, we need an extract function that will have a