Merge branch 'topic/testsuite/add_only_rewriting_flag' into 'master'

Add the "--only-with-auto-fix" testsuite flag

Closes #474

See merge request eng/libadalang/langkit-query-language!438

Author: HugoGGuerrier

testsuite/testsuite.py

@@ -7,7 +7,7 @@
 import statistics
 import subprocess
 import sys
-from typing import TextIO, Callable
+from typing import TextIO, Callable, Any
 
 from e3.fs import mkdir, rm
 from e3.testsuite import Testsuite, logger, TestsuiteCore
@@ -20,48 +20,63 @@
     benchmarks_driver, refactor_driver
 )
 
-class PerfTestFinder(YAMLTestFinder):
+
+class StandardTestFinder(YAMLTestFinder):
     """
-    Testcase finder to use in perf mode.
+    Testcase finder to use in stadard mode.
 
-    This finder automatically discard tests that do not have performance
-    measuring instructions. This is preferable to creating these tests but
-    skipping them (SKIP status) as most tests do not support performance
-    measurements: less noise in the testsuite report.
+    This finder exclude test cases from the 'tests/perf/' directory to avoid
+    running them in standard mode. This allow performance exclusive test
+    cases.
+    This finder doesn't exclude all performance compatible tests because
+    we want to be able to write baseline/performance hybrid tests.
     """
 
+    def __init__(
+            self,
+            env_constraints: dict[str, Callable[[Any], bool]] | None = None
+        ):
+        """
+        Create a new standard test finder, with a given list of constraints to
+        check on the test environment.
+        """
+        super().__init__()
+        self.env_constraints = env_constraints
+
+    def verify_then_return(self, test: TestFinderResult) -> TestFinderResult | None:
+        """
+        Check that the provided test environment is validating all constraints
+        attached to this test finder.
+        """
+        if self.env_constraints:
+            for field, predicate in self.env_constraints.items():
+                if not predicate(test.test_env.get(field, None)):
+                    return None
+        return test
+
     def probe(self,
               testsuite: TestsuiteCore,
               dirpath: str,
               dirnames: list[str],
-              filenames: list[str]) -> TestFinderResult:
+              filenames: list[str]) -> TestFinderResult | None:
         # Probe testcases as usual
         result = super().probe(testsuite, dirpath, dirnames, filenames)
 
-        # Reject testcases which do not contain performance measuring
-        # instructions.
-        if result is None or P.join("tests", "perf") not in result.test_dir:
+        # Reject all tests which have 'tests/perf' in their directory name
+        if result is None or P.join("tests", "perf") in result.test_dir:
             return None
 
-        # Make sure that the driver supports performance measuring
-        if not result.driver_cls.perf_supported:
-            raise ProbingError(
-                f"The '{result.driver_cls.__name__}' driver does not support"
-                 " performance measuring"
-            )
+        return self.verify_then_return(result)
 
-        return result
 
-
-class StandardTestFinder(YAMLTestFinder):
+class PerfTestFinder(StandardTestFinder):
     """
-    Testcase finder to use in stadard mode.
+    Testcase finder to use in perf mode.
 
-    This finder exclude test cases from the 'tests/perf/' directory to avoid
-    running them in standard mode. This allow performance exclusive test
-    cases.
-    This finder doesn't exclude all performance compatible tests because
-    we want to be able to write baseline/performance hybrid tests.
+    This finder automatically discard tests that do not have performance
+    measuring instructions. This is preferable to creating these tests but
+    skipping them (SKIP status) as most tests do not support performance
+    measurements: less noise in the testsuite report.
     """
 
     def probe(self,
@@ -72,11 +87,19 @@ def probe(self,
         # Probe testcases as usual
         result = super().probe(testsuite, dirpath, dirnames, filenames)
 
-        # Reject all tests which have 'tests/perf' in their directory name
-        if result is None or P.join("tests", "perf") in result.test_dir:
+        # Reject testcases which do not contain performance measuring
+        # instructions.
+        if result is None or P.join("tests", "perf") not in result.test_dir:
             return None
 
-        return result
+        # Make sure that the driver supports performance measuring
+        if not result.driver_cls.perf_supported:
+            raise ProbingError(
+                f"The '{result.driver_cls.__name__}' driver does not support"
+                 " performance measuring"
+            )
+
+        return self.verify_then_return(result)
 
 
 class LKQLTestsuite(Testsuite):
@@ -137,12 +160,24 @@ def add_options(self, parser: ArgumentParser) -> None:
                  ' to get feedback quickly during development.'
         )
 
+        parser.add_argument(
+            '--only-with-auto-fix',
+            action='store_true',
+            help='Run only tests that uses the LKQL rewriting API through'
+                 ' checkers auto-fixing function.'
+        )
+
     @property
     def test_finders(self) -> list[TestFinder]:
+        # Create the test environment constraint list
+        env_constraints = dict()
+        if self.env.options.only_with_auto_fix:
+            env_constraints["auto_fix"] = lambda v: v == True
+
         return [
-            PerfTestFinder()
+            PerfTestFinder(env_constraints)
             if self.env.perf_mode else
-            StandardTestFinder()
+            StandardTestFinder(env_constraints)
         ]
 
     def set_up(self) -> None: