twister: count retries after failures or errors

When twister is set to retry any failures, count the number of retries
and record the number in the json file. This will help us identify
unstable tests or tests requiring attention.

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

scripts/pylib/twister/twisterlib/reports.py

@@ -267,6 +267,8 @@ def json_report(self, filename, version="NA"):
             if rom_size:
                 suite["rom_size"] = rom_size
 
+            suite['retries'] = instance.retries
+
             if instance.status in ["error", "failed"]:
                 suite['status'] = instance.status
                 suite["reason"] = instance.reason

scripts/pylib/twister/twisterlib/runner.py

@@ -972,6 +972,9 @@ def add_tasks_to_queue(self, pipeline, build_only=False, test_only=False, retry_
 
             if instance.status not in no_retry_statuses:
                 logger.debug(f"adding {instance.name}")
+                if instance.status:
+                    instance.retries += 1
+
                 instance.status = None
                 if test_only and instance.run:
                     pipeline.put({"op": "run", "test": instance})

scripts/pylib/twister/twisterlib/testinstance.py

@@ -40,6 +40,7 @@ def __init__(self, testsuite, platform, outdir):
         self.handler = None
         self.outdir = outdir
         self.execution_time = 0
+        self.retries = 0
 
         self.name = os.path.join(platform.name, testsuite.name)
         self.run_id = self._get_run_id()

scripts/pylib/twister/twisterlib/testplan.py

@@ -524,6 +524,7 @@ def load_from_file(self, file, filter_platform=[]):
                 if status in ["error", "failed"]:
                     instance.status = None
                     instance.reason = None
+                    instance.retries += 1
                 # test marked as passed (built only) but can run when
                 # --test-only is used. Reset status to capture new results.
                 elif status == 'passed' and instance.run and self.options.test_only: