Skip to content

Commit 345eb8e

Browse files
LukaszMrugalacarlescufi
LukaszMrugala
authored and
carlescufi
committed
scripts: twister: Elevate Status Error
Status errors previously logged an error, but didn't fail the running test. This commit changes that and introduces a new StatusAttributeError to use there. One test is modified so it follows proper status form. One test for the new error has been added. Status errors now will properly mark the Instance as ERROR and not run TestCases as SKIP. This necessitated some code layout changes in runner.py Signed-off-by: Lukasz Mrugala <[email protected]>
1 parent 8b50d37 commit 345eb8e

File tree

7 files changed

+202
-109
lines changed

7 files changed

+202
-109
lines changed

‎scripts/pylib/twister/twisterlib/error.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,9 @@ class BuildError(TwisterException):
2222

2323
class ExecutionError(TwisterException):
2424
pass
25+
26+
27+
class StatusAttributeError(TwisterException):
28+
def __init__(self, cls : type, value):
29+
msg = f'{cls.__name__} assigned status {value}, which could not be cast to a TwisterStatus.'
30+
super().__init__(msg)

‎scripts/pylib/twister/twisterlib/harness.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
from pytest import ExitCode
2020
from twisterlib.reports import ReportStatus
21-
from twisterlib.error import ConfigurationError
21+
from twisterlib.error import ConfigurationError, StatusAttributeError
2222
from twisterlib.environment import ZEPHYR_BASE, PYTEST_PLUGIN_INSTALLED
2323
from twisterlib.handlers import Handler, terminate_process, SUPPORTED_SIMS_IN_PYTEST
2424
from twisterlib.statuses import TwisterStatus
@@ -76,9 +76,7 @@ def status(self, value : TwisterStatus) -> None:
7676
key = value.name if isinstance(value, Enum) else value
7777
self._status = TwisterStatus[key]
7878
except KeyError:
79-
logger.error(f'Harness assigned status "{value}"'
80-
f' without an equivalent in TwisterStatus.'
81-
f' Assignment was ignored.')
79+
raise StatusAttributeError(self.__class__, value)
8280

8381
def configure(self, instance):
8482
self.instance = instance

‎scripts/pylib/twister/twisterlib/runner.py

Lines changed: 175 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
from domains import Domains
2626
from twisterlib.cmakecache import CMakeCache
2727
from twisterlib.environment import canonical_zephyr_base
28-
from twisterlib.error import BuildError, ConfigurationError
28+
from twisterlib.error import BuildError, ConfigurationError, StatusAttributeError
2929
from twisterlib.statuses import TwisterStatus
3030

3131
import elftools
@@ -601,130 +601,212 @@ def log_info_file(self, inline_logs):
601601
self.log_info("{}".format(b_log), inline_logs)
602602

603603

604+
def _add_to_pipeline(self, pipeline, op: str, additionals: dict={}):
605+
try:
606+
if op:
607+
task = dict({'op': op, 'test': self.instance}, **additionals)
608+
pipeline.put(task)
609+
# Only possible RuntimeError source here is a mutation of the pipeline during iteration.
610+
# If that happens, we ought to consider the whole pipeline corrupted.
611+
except RuntimeError as e:
612+
logger.error(f"RuntimeError: {e}")
613+
traceback.print_exc()
614+
615+
604616
def process(self, pipeline, done, message, lock, results):
605617
op = message.get('op')
606618

607619
self.instance.setup_handler(self.env)
608620

609621
if op == "filter":
610-
ret = self.cmake(filter_stages=self.instance.filter_stages)
611-
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
612-
pipeline.put({"op": "report", "test": self.instance})
613-
else:
614-
# Here we check the dt/kconfig filter results coming from running cmake
615-
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
616-
logger.debug("filtering %s" % self.instance.name)
617-
self.instance.status = TwisterStatus.FILTER
618-
self.instance.reason = "runtime filter"
619-
results.skipped_runtime += 1
620-
self.instance.add_missing_case_status(TwisterStatus.SKIP)
621-
pipeline.put({"op": "report", "test": self.instance})
622+
try:
623+
ret = self.cmake(filter_stages=self.instance.filter_stages)
624+
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
625+
next_op = 'report'
622626
else:
623-
pipeline.put({"op": "cmake", "test": self.instance})
627+
# Here we check the dt/kconfig filter results coming from running cmake
628+
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
629+
logger.debug("filtering %s" % self.instance.name)
630+
self.instance.status = TwisterStatus.FILTER
631+
self.instance.reason = "runtime filter"
632+
results.skipped_runtime += 1
633+
self.instance.add_missing_case_status(TwisterStatus.SKIP)
634+
next_op = 'report'
635+
else:
636+
next_op = 'cmake'
637+
except StatusAttributeError as sae:
638+
logger.error(str(sae))
639+
self.instance.status = TwisterStatus.ERROR
640+
reason = 'Incorrect status assignment'
641+
self.instance.reason = reason
642+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
643+
next_op = 'report'
644+
finally:
645+
self._add_to_pipeline(pipeline, next_op)
624646

625647
# The build process, call cmake and build with configured generator
626648
elif op == "cmake":
627-
ret = self.cmake()
628-
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
629-
pipeline.put({"op": "report", "test": self.instance})
630-
elif self.options.cmake_only:
631-
if self.instance.status == TwisterStatus.NONE:
632-
self.instance.status = TwisterStatus.PASS
633-
pipeline.put({"op": "report", "test": self.instance})
634-
else:
635-
# Here we check the runtime filter results coming from running cmake
636-
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
637-
logger.debug("filtering %s" % self.instance.name)
638-
self.instance.status = TwisterStatus.FILTER
639-
self.instance.reason = "runtime filter"
640-
results.skipped_runtime += 1
641-
self.instance.add_missing_case_status(TwisterStatus.SKIP)
642-
pipeline.put({"op": "report", "test": self.instance})
649+
try:
650+
ret = self.cmake()
651+
if self.instance.status in [TwisterStatus.FAIL, TwisterStatus.ERROR]:
652+
next_op = 'report'
653+
elif self.options.cmake_only:
654+
if self.instance.status == TwisterStatus.NONE:
655+
self.instance.status = TwisterStatus.PASS
656+
next_op = 'report'
643657
else:
644-
pipeline.put({"op": "build", "test": self.instance})
658+
# Here we check the runtime filter results coming from running cmake
659+
if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
660+
logger.debug("filtering %s" % self.instance.name)
661+
self.instance.status = TwisterStatus.FILTER
662+
self.instance.reason = "runtime filter"
663+
results.skipped_runtime += 1
664+
self.instance.add_missing_case_status(TwisterStatus.SKIP)
665+
next_op = 'report'
666+
else:
667+
next_op = 'build'
668+
except StatusAttributeError as sae:
669+
logger.error(str(sae))
670+
self.instance.status = TwisterStatus.ERROR
671+
reason = 'Incorrect status assignment'
672+
self.instance.reason = reason
673+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
674+
next_op = 'report'
675+
finally:
676+
self._add_to_pipeline(pipeline, next_op)
645677

646678
elif op == "build":
647-
logger.debug("build test: %s" % self.instance.name)
648-
ret = self.build()
649-
if not ret:
650-
self.instance.status = TwisterStatus.ERROR
651-
self.instance.reason = "Build Failure"
652-
pipeline.put({"op": "report", "test": self.instance})
653-
else:
654-
# Count skipped cases during build, for example
655-
# due to ram/rom overflow.
656-
if self.instance.status == TwisterStatus.SKIP:
657-
results.skipped_runtime += 1
658-
self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)
659-
660-
if ret.get('returncode', 1) > 0:
661-
self.instance.add_missing_case_status(TwisterStatus.BLOCK, self.instance.reason)
662-
pipeline.put({"op": "report", "test": self.instance})
679+
try:
680+
logger.debug("build test: %s" % self.instance.name)
681+
ret = self.build()
682+
if not ret:
683+
self.instance.status = TwisterStatus.ERROR
684+
self.instance.reason = "Build Failure"
685+
next_op = 'report'
663686
else:
664-
if self.instance.testsuite.harness in ['ztest', 'test']:
665-
logger.debug(f"Determine test cases for test instance: {self.instance.name}")
666-
try:
667-
self.determine_testcases(results)
668-
pipeline.put({"op": "gather_metrics", "test": self.instance})
669-
except BuildError as e:
670-
logger.error(str(e))
671-
self.instance.status = TwisterStatus.ERROR
672-
self.instance.reason = str(e)
673-
pipeline.put({"op": "report", "test": self.instance})
687+
# Count skipped cases during build, for example
688+
# due to ram/rom overflow.
689+
if self.instance.status == TwisterStatus.SKIP:
690+
results.skipped_runtime += 1
691+
self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)
692+
693+
if ret.get('returncode', 1) > 0:
694+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, self.instance.reason)
695+
next_op = 'report'
674696
else:
675-
pipeline.put({"op": "gather_metrics", "test": self.instance})
697+
if self.instance.testsuite.harness in ['ztest', 'test']:
698+
logger.debug(f"Determine test cases for test instance: {self.instance.name}")
699+
try:
700+
self.determine_testcases(results)
701+
next_op = 'gather_metrics'
702+
except BuildError as e:
703+
logger.error(str(e))
704+
self.instance.status = TwisterStatus.ERROR
705+
self.instance.reason = str(e)
706+
next_op = 'report'
707+
else:
708+
next_op = 'gather_metrics'
709+
except StatusAttributeError as sae:
710+
logger.error(str(sae))
711+
self.instance.status = TwisterStatus.ERROR
712+
reason = 'Incorrect status assignment'
713+
self.instance.reason = reason
714+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
715+
next_op = 'report'
716+
finally:
717+
self._add_to_pipeline(pipeline, next_op)
676718

677719
elif op == "gather_metrics":
678-
ret = self.gather_metrics(self.instance)
679-
if not ret or ret.get('returncode', 1) > 0:
720+
try:
721+
ret = self.gather_metrics(self.instance)
722+
if not ret or ret.get('returncode', 1) > 0:
723+
self.instance.status = TwisterStatus.ERROR
724+
self.instance.reason = "Build Failure at gather_metrics."
725+
next_op = 'report'
726+
elif self.instance.run and self.instance.handler.ready:
727+
next_op = 'run'
728+
else:
729+
next_op = 'report'
730+
except StatusAttributeError as sae:
731+
logger.error(str(sae))
680732
self.instance.status = TwisterStatus.ERROR
681-
self.instance.reason = "Build Failure at gather_metrics."
682-
pipeline.put({"op": "report", "test": self.instance})
683-
elif self.instance.run and self.instance.handler.ready:
684-
pipeline.put({"op": "run", "test": self.instance})
685-
else:
686-
pipeline.put({"op": "report", "test": self.instance})
733+
reason = 'Incorrect status assignment'
734+
self.instance.reason = reason
735+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
736+
next_op = 'report'
737+
finally:
738+
self._add_to_pipeline(pipeline, next_op)
687739

688740
# Run the generated binary using one of the supported handlers
689741
elif op == "run":
690-
logger.debug("run test: %s" % self.instance.name)
691-
self.run()
692-
logger.debug(f"run status: {self.instance.name} {self.instance.status}")
693742
try:
743+
logger.debug("run test: %s" % self.instance.name)
744+
self.run()
745+
logger.debug(f"run status: {self.instance.name} {self.instance.status}")
746+
694747
# to make it work with pickle
695748
self.instance.handler.thread = None
696749
self.instance.handler.duts = None
697-
pipeline.put({
698-
"op": "report",
699-
"test": self.instance,
750+
751+
next_op = 'report'
752+
additionals = {
700753
"status": self.instance.status,
701754
"reason": self.instance.reason
702-
}
703-
)
704-
except RuntimeError as e:
705-
logger.error(f"RuntimeError: {e}")
706-
traceback.print_exc()
755+
}
756+
except StatusAttributeError as sae:
757+
logger.error(str(sae))
758+
self.instance.status = TwisterStatus.ERROR
759+
reason = 'Incorrect status assignment'
760+
self.instance.reason = reason
761+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
762+
next_op = 'report'
763+
additionals = {}
764+
finally:
765+
self._add_to_pipeline(pipeline, next_op, additionals)
707766

708767
# Report results and output progress to screen
709768
elif op == "report":
710-
with lock:
711-
done.put(self.instance)
712-
self.report_out(results)
713-
714-
if not self.options.coverage:
715-
if self.options.prep_artifacts_for_testing:
716-
pipeline.put({"op": "cleanup", "mode": "device", "test": self.instance})
717-
elif self.options.runtime_artifact_cleanup == "pass" and self.instance.status == TwisterStatus.PASS:
718-
pipeline.put({"op": "cleanup", "mode": "passed", "test": self.instance})
719-
elif self.options.runtime_artifact_cleanup == "all":
720-
pipeline.put({"op": "cleanup", "mode": "all", "test": self.instance})
769+
try:
770+
with lock:
771+
done.put(self.instance)
772+
self.report_out(results)
773+
774+
next_op = None
775+
additionals = {}
776+
if not self.options.coverage:
777+
if self.options.prep_artifacts_for_testing:
778+
next_op = 'cleanup'
779+
additionals = {"mode": "device"}
780+
elif self.options.runtime_artifact_cleanup == "pass" and self.instance.status == TwisterStatus.PASS:
781+
next_op = 'cleanup'
782+
additionals = {"mode": "passed"}
783+
elif self.options.runtime_artifact_cleanup == "all":
784+
next_op = 'cleanup'
785+
additionals = {"mode": "all"}
786+
except StatusAttributeError as sae:
787+
logger.error(str(sae))
788+
self.instance.status = TwisterStatus.ERROR
789+
reason = 'Incorrect status assignment'
790+
self.instance.reason = reason
791+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
792+
next_op = None
793+
additionals = {}
794+
finally:
795+
self._add_to_pipeline(pipeline, next_op, additionals)
721796

722797
elif op == "cleanup":
723-
mode = message.get("mode")
724-
if mode == "device":
725-
self.cleanup_device_testing_artifacts()
726-
elif mode == "passed" or (mode == "all" and self.instance.reason != "Cmake build failure"):
727-
self.cleanup_artifacts()
798+
try:
799+
mode = message.get("mode")
800+
if mode == "device":
801+
self.cleanup_device_testing_artifacts()
802+
elif mode == "passed" or (mode == "all" and self.instance.reason != "Cmake build failure"):
803+
self.cleanup_artifacts()
804+
except StatusAttributeError as sae:
805+
logger.error(str(sae))
806+
self.instance.status = TwisterStatus.ERROR
807+
reason = 'Incorrect status assignment'
808+
self.instance.reason = reason
809+
self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
728810

729811
def determine_testcases(self, results):
730812
yaml_testsuite_name = self.instance.testsuite.id

‎scripts/pylib/twister/twisterlib/testinstance.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
from twisterlib.testsuite import TestCase, TestSuite
1717
from twisterlib.platform import Platform
18-
from twisterlib.error import BuildError
18+
from twisterlib.error import BuildError, StatusAttributeError
1919
from twisterlib.size_calc import SizeCalculator
2020
from twisterlib.statuses import TwisterStatus
2121
from twisterlib.handlers import (
@@ -105,9 +105,7 @@ def status(self, value : TwisterStatus) -> None:
105105
key = value.name if isinstance(value, Enum) else value
106106
self._status = TwisterStatus[key]
107107
except KeyError:
108-
logger.error(f'TestInstance assigned status "{value}"'
109-
f' without an equivalent in TwisterStatus.'
110-
f' Assignment was ignored.')
108+
raise StatusAttributeError(self.__class__, value)
111109

112110
def add_filter(self, reason, filter_type):
113111
self.filters.append({'type': filter_type, 'reason': reason })

‎scripts/pylib/twister/twisterlib/testsuite.py

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
from twisterlib.mixins import DisablePyTestCollectionMixin
1717
from twisterlib.environment import canonical_zephyr_base
18-
from twisterlib.error import TwisterException, TwisterRuntimeError
18+
from twisterlib.error import StatusAttributeError, TwisterException, TwisterRuntimeError
1919
from twisterlib.statuses import TwisterStatus
2020

2121
logger = logging.getLogger('twister')
@@ -377,9 +377,7 @@ def status(self, value : TwisterStatus) -> None:
377377
key = value.name if isinstance(value, Enum) else value
378378
self._status = TwisterStatus[key]
379379
except KeyError:
380-
logger.error(f'TestCase assigned status "{value}"'
381-
f' without an equivalent in TwisterStatus.'
382-
f' Assignment was ignored.')
380+
raise StatusAttributeError(self.__class__, value)
383381

384382
def __lt__(self, other):
385383
return self.name < other.name
@@ -445,9 +443,7 @@ def status(self, value : TwisterStatus) -> None:
445443
key = value.name if isinstance(value, Enum) else value
446444
self._status = TwisterStatus[key]
447445
except KeyError:
448-
logger.error(f'TestSuite assigned status "{value}"'
449-
f' without an equivalent in TwisterStatus.'
450-
f' Assignment was ignored.')
446+
raise StatusAttributeError(self.__class__, value)
451447

452448
def load(self, data):
453449
for k, v in data.items():

0 commit comments

Comments
 (0)