make tests fail when criteria not found

When criteria is not available, performance tests pass without any
errors. This commit makes tests fail in such situation. It lets us know
that performance baselines are missing.

Signed-off-by: Takahiro Itazuri <[email protected]>

Author: zulinx86

tests/framework/stats/consumer.py

@@ -7,7 +7,8 @@
 from numbers import Number
 from typing import Any, Callable
 from collections import defaultdict
-from framework.utils import ExceptionAggregator
+from framework.utils import ExceptionAggregator, get_kernel_version
+from framework.utils_cpuid import get_cpu_model_name, get_instance_type
 
 from .criteria import CriteriaException
 from .metadata import Provider as MetadataProvider
@@ -141,6 +142,16 @@ def process(self, fail_fast=False) -> (dict, dict):
                         self._failure_aggregator.add_row(fail_msg)
                         if fail_fast:
                             raise self._failure_aggregator
+                else:
+                    self._statistics[ms_name][st_def.name]["outcome"] = "FAILED"
+                    fail_msg = (
+                        f"'{ms_name}/{st_def.name}': Criteria not found for "
+                        f"{get_instance_type()} / {get_cpu_model_name()} / "
+                        f"kernel {get_kernel_version(1)}."
+                    )
+                    self._failure_aggregator.add_row(fail_msg)
+                    if fail_fast:
+                        raise self._failure_aggregator
 
         self._reset()