sanitycheck: fixed handling of retries

When re-running failed tests, do not go through any filtering, just load
the failed tests directly and execute them. The filtering was done based
on default platforms and any tests that were failing on non-default
platforms were ignored.

Fixes #13956

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

scripts/sanitycheck

@@ -1814,17 +1814,17 @@ class TestInstance:
     def create_overlay(self, platform):
         file = os.path.join(self.outdir, "overlay.conf")
         os.makedirs(self.outdir, exist_ok=True)
-        f = open(file, "w")
-        content = ""
+        with open(file, "w") as f:
+            content = ""
 
-        if len(self.test.extra_configs) > 0:
-            content = "\n".join(self.test.extra_configs)
-        if options.enable_coverage:
-            if platform in options.coverage_platform:
-                content = content + "\nCONFIG_COVERAGE=y"
+            if len(self.test.extra_configs) > 0:
+                content = "\n".join(self.test.extra_configs)
 
-        f.write(content)
-        f.close()
+            if options.enable_coverage:
+                if platform in options.coverage_platform:
+                    content = content + "\nCONFIG_COVERAGE=y"
+
+            f.write(content)
 
     def calculate_sizes(self):
         """Get the RAM/ROM sizes of a test case.
@@ -1942,8 +1942,15 @@ class TestSuite:
 
         self.instances = {}
 
-    def get_last_failed(self):
+    def get_platform(self, name):
+        selected_platform = None
+        for platform in self.platforms:
+            if platform.name == name:
+                selected_platform = platform
+                break
+        return selected_platform
 
+    def get_last_failed(self):
         try:
             if not os.path.exists(LAST_SANITY):
                 raise SanityRuntimeError("Couldn't find last sanity run.")
@@ -1954,13 +1961,16 @@ class TestSuite:
         result = []
         with open(LAST_SANITY, "r") as fp:
             cr = csv.DictReader(fp)
+            instance_list = []
             for row in cr:
                 if row["passed"] == "True":
                     continue
                 test = row["test"]
-                platform = row["platform"]
-                result.append((test, platform))
-        return result
+                platform = self.get_platform(row["platform"])
+                instance = TestInstance(self.testcases[test], platform, self.outdir)
+                instance.create_overlay(platform.name)
+                instance_list.append(instance)
+            self.add_instances(instance_list)
 
     def load_from_file(self, file):
         try:
@@ -1976,14 +1986,9 @@ class TestSuite:
             instance_list = []
             for row in cr:
                 name = os.path.join(row[0], row[1])
-                platforms = self.arches[row[3]].platforms
-                myp = None
-                for platform in platforms:
-                    if platform.name == row[2]:
-                        selected_platform = platform
-                        break
-                instance = TestInstance(self.testcases[name], selected_platform, self.outdir)
-                instance.create_overlay(selected_platform.name)
+                platform = self.get_platform(row[2])
+                instance = TestInstance(self.testcases[name], platform, self.outdir)
+                instance.create_overlay(platform.name)
                 instance_list.append(instance)
             self.add_instances(instance_list)
 
@@ -2007,7 +2012,6 @@ class TestSuite:
         instances = []
         discards = {}
         platform_filter = options.platform
-        last_failed = options.only_failed
         testcase_filter = run_individual_tests
         arch_filter = options.arch
         tag_filter = options.tag
@@ -2022,9 +2026,6 @@ class TestSuite:
         verbose("    exclude_tag: " + str(exclude_tag))
         verbose("  config_filter: " + str(config_filter))
 
-        if last_failed:
-            failed_tests = self.get_last_failed()
-
         default_platforms = False
 
         if all_plats:
@@ -2062,10 +2063,6 @@ class TestSuite:
                     if testcase_filter and tc_name not in testcase_filter:
                         continue
 
-                    if last_failed and (
-                            tc.name, plat.name) not in failed_tests:
-                        continue
-
                     if arch_filter and arch_name not in arch_filter:
                         continue
 
@@ -2214,11 +2211,6 @@ class TestSuite:
                         discards[instance] = "Testcase name filter"
                         continue
 
-                    if last_failed and (
-                            tc.name, plat.name) not in failed_tests:
-                        discards[instance] = "Passed or skipped during last run"
-                        continue
-
                     if arch_filter and arch_name not in arch_filter:
                         discards[instance] = "Command line testcase arch filter"
                         continue
@@ -3240,6 +3232,8 @@ def main():
     discards = []
     if options.load_tests:
         ts.load_from_file(options.load_tests)
+    elif options.only_failed:
+        ts.get_last_failed()
     else:
         discards = ts.apply_filters()