Merge branch 'topic/testsuite/fix_flag_checking' into 'master'

Fix and enhance the flag checking in the testsuite

Closes #479

See merge request eng/libadalang/langkit-query-language!488

Author: HugoGGuerrier

testsuite/drivers/base_driver.py

@@ -12,6 +12,7 @@ class BenchmarksDriver(BaseDriver):
     """
 
     perf_supported = True
+    flag_checking_supported = False
 
     @property
     def baseline(self) -> tuple[str, str, bool]:

testsuite/drivers/benchmarks_driver.py

@@ -1,6 +1,6 @@
 import re
 
-from drivers.base_driver import BaseDriver, Flags
+from drivers.base_driver import BaseDriver, TaggedLines
 
 
 class CheckerDriver(BaseDriver):
@@ -28,6 +28,10 @@ class CheckerDriver(BaseDriver):
     perf_supported = True
     flag_checking_supported = True
 
+    _flag_line_pattern = re.compile(
+        rf"^({BaseDriver.ada_file_pattern}):(\d+):\d+: rule violation: .*$"
+    )
+
     def run(self) -> None:
         args = []
 
@@ -71,7 +75,7 @@ def run(self) -> None:
                 assert auto_fix_mode in ["DISPLAY", "NEW_FILE", "PATCH_FILE"]
                 self.check_run(
                     self.lkql_fix_exe + args + ['--auto-fix-mode', auto_fix_mode],
-                    parse_flags=False,
+                    check_flags=False,
                     catch_error=False,
                 )
 
@@ -92,21 +96,18 @@ def run(self) -> None:
                             self.output += f"=== {pf} content:\n"
                             self.output += f.read()
 
-    def parse_flagged_lines(self, output: str) -> Flags:
-        # Compile the pattern to match a checker output
-        pattern = re.compile(
-            r"^([a-zA-Z][a-zA-Z0-9_\-]*\.(adb|ads)):(\d+):\d+: rule violation: .*$"
-        )
-
+    def parse_flagged_lines(self, output: str) -> dict[str, TaggedLines]:
         # Prepare the result
-        res = Flags()
+        res: dict[str, TaggedLines] = {}
 
         # For each line of the output search the groups in the line
         for line in output.splitlines():
-            search_result = pattern.search(line)
+            search_result = self._flag_line_pattern.search(line)
             if search_result is not None:
                 (file, _, line_num) = search_result.groups()
-                res.add_flag(file, int(line_num))
+                if not res.get(file):
+                    res[file] = TaggedLines()
+                res[file].tag_line(int(line_num))
 
         # Return the result
         return res

testsuite/drivers/checker_driver.py

@@ -3,80 +3,7 @@
 import re
 import xml.etree.ElementTree as ET
 
-from drivers.base_driver import BaseDriver, Flags
-
-
-# --- GNATcheck output parsing functions
-
-_flag_line_pattern = re.compile(
-    r"^([a-zA-Z][a-zA-Z0-9_\.\-]*\.(adb|ads|ada|ada_spec)):(\d+):\d+: .*$"
-)
-
-def _parse_full(output: str) -> Flags:
-    """
-    Parse the full formatted gnatcheck output.
-    """
-    # Prepare the result
-    res = Flags()
-
-    # Parse the gnatcheck full output
-    is_parsing = False
-    for line in output.splitlines():
-        if not is_parsing:
-            is_parsing = "2. Exempted Coding Standard Violations" in line
-        else:
-            search_result = _flag_line_pattern.search(line)
-            if search_result is not None:
-                (file, _, line_num) = search_result.groups()
-                res.add_flag(file, int(line_num))
-            is_parsing = "5. Language violations" not in line
-
-    # Return the result
-    return res
-
-def _parse_short_and_brief(output: str) -> Flags:
-    """
-    Parse the short formatted gnatcheck output.
-    """
-    # Prepare the result
-    res = Flags()
-
-    # Parse the output
-    for line in output.splitlines():
-        search_result = _flag_line_pattern.search(line)
-        if search_result is not None:
-            (file, _, line_num) = search_result.groups()
-            res.add_flag(file, int(line_num))
-
-    # Return the result
-    return res
-
-def _parse_xml(output: str) -> Flags:
-    """
-    Parse the xml formatted gnatcheck output.
-    """
-    # Prepare the result
-    res = Flags()
-
-    # Parse the xml result
-    xml_tree = ET.fromstring(output)
-    violations = xml_tree.find("violations")
-
-    # If the "violations" tag exists in the output, parse it as a full XML output
-    if violations is not None:
-        for violation in violations:
-            file, line_num = violation.attrib["file"], int(violation.attrib["line"])
-            res.add_flag(file, line_num)
-
-    # Else the ouput is a brief XML one
-    else:
-        for elem in xml_tree.findall("*"):
-            if elem.tag in ("violation", "exemption-problem", "exempted-violation"):
-                file, line_num = elem.attrib["file"], int(elem.attrib["line"])
-                res.add_flag(file, line_num)
-
-    # Return the result
-    return res
+from drivers.base_driver import BaseDriver, TaggedLines
 
 
 class GnatcheckDriver(BaseDriver):
@@ -191,12 +118,86 @@ class GnatcheckDriver(BaseDriver):
         "gnatkp": "gnatkp"
     }
     output_formats = set(['brief', 'full', 'short', 'xml'])
-    parsers = {
-        'full': _parse_full,
-        'short': _parse_short_and_brief,
-        'brief': _parse_short_and_brief,
-        'xml': _parse_xml
-    }
+
+    flag_line_pattern = re.compile(
+        rf"^({BaseDriver.ada_file_pattern}):(\d+):\d+: (rule violation|warning|error): .*$"
+    )
+
+    @classmethod
+    def _parse_full(cls, output: str) -> dict[str, TaggedLines]:
+        """
+        Parse the full formatted GNATcheck output.
+        """
+        # Remove useless parts of the full output
+        try:
+            output = output[output.index("2. Exempted Coding Standard Violations"):]
+            output = output[:output.index("5. Language violations")]
+
+            # Then use the "short" parser since we remove the useless part of the
+            # "full" format.
+            return cls._parse_short_and_brief(output)
+        except ValueError:
+            # When catching this error, it means that the `index` method failed
+            # to find required textual bounds.
+            return {}
+
+    @classmethod
+    def _parse_short_and_brief(cls, output: str) -> dict[str, TaggedLines]:
+        """
+        Parse the short formatted GNATcheck output.
+        """
+        res: dict[str, TaggedLines] = {}
+
+        for line in output.splitlines():
+            search_result = cls.flag_line_pattern.search(line)
+            if search_result is not None:
+                (file, _, line_num, _) = search_result.groups()
+                if not res.get(file):
+                    res[file] = TaggedLines()
+                res[file].tag_line(int(line_num))
+
+        return res
+
+    @classmethod
+    def _parse_xml(cls, output: str) -> dict[str, TaggedLines]:
+        """
+        Parse the XML formatted GNATcheck output.
+        """
+        res: dict[str, TaggedLines] = {}
+
+        def tag_line(file: str, line: int):
+            if not res.get(file):
+                res[file] = TaggedLines()
+            res[file].tag_line(line)
+
+        # Parse the xml result
+        xml_tree = ET.fromstring(output)
+        violations = xml_tree.find("violations")
+
+        # If the "violations" tag exists in the output, parse it as a full XML output
+        if violations is not None:
+            for violation in violations:
+                file, line_num = violation.attrib["file"], int(violation.attrib["line"])
+                tag_line(file, line_num)
+
+        # Else the output is a brief XML one
+        else:
+            for elem in xml_tree.findall("*"):
+                if elem.tag in ("violation", "exemption-problem", "exempted-violation"):
+                    file, line_num = elem.attrib["file"], int(elem.attrib["line"])
+                    tag_line(file, line_num)
+
+        # Return the result
+        return res
+
+    @classmethod
+    def parsers(cls):
+        return {
+            'full': cls._parse_full,
+            'short': cls._parse_short_and_brief,
+            'brief': cls._parse_short_and_brief,
+            'xml': cls._parse_xml
+        }
 
     @property
     def default_process_timeout(self):
@@ -262,7 +263,11 @@ def cat(
             exec(global_python, globs, locs)
 
         # Prepare the execution flagged lines as an empty object
-        flagged_lines = Flags()
+        files_flagged_lines: dict[str, TaggedLines] = {}
+
+        def add_to_files_flagged_lines(to_add: dict[str, TaggedLines]):
+            for file, tagged_lines in to_add.items():
+                files_flagged_lines[file] = files_flagged_lines.get(file, TaggedLines()).combine(tagged_lines)
 
         def capture_exec_python(code: str) -> None:
             """
@@ -466,7 +471,7 @@ def run_one_test(test_data: dict[str, any]) -> None:
 
                 # Get the lines flagged by the test running and add it to all flagged lines
                 if self.flag_checking and parse_output_for_flags:
-                    flagged_lines.add_other_flags(
+                    add_to_files_flagged_lines(
                         self.parse_flagged_lines(
                             (
                                 exec_output + report_file_content
@@ -521,8 +526,8 @@ def run_one_test(test_data: dict[str, any]) -> None:
 
         # Check the execution flagged lines
         if self.flag_checking:
-            self.check_flags(flagged_lines)
+            self.check_flags(files_flagged_lines)
 
-    def parse_flagged_lines(self, output: str, format: str) -> Flags:
+    def parse_flagged_lines(self, output: str, format: str) -> dict[str, TaggedLines]:
         assert format in self.output_formats
-        return self.parsers[format](output)
+        return self.parsers()[format](output)

testsuite/drivers/gnatcheck_driver.py

@@ -19,6 +19,7 @@ class InterpreterDriver(BaseDriver):
     """
 
     perf_supported = True
+    flag_checking_supported = False
 
     def run(self) -> None:
         # Build the process's arguments list

testsuite/drivers/interpreter_driver.py

@@ -14,6 +14,9 @@ class JavaDriver(BaseDriver):
     in the test directory.
     """
 
+    perf_supported = False
+    flag_checking_supported = False
+
     def run(self) -> None:
         # Get and check the test Java main file
         main_java_file = self.working_dir("Main.java")

testsuite/drivers/java_driver.py

@@ -13,6 +13,10 @@ class ParserDriver(BaseDriver):
     Test arguments:
         - rule: name of the grammar rule to pass to `parse`
     """
+
+    perf_supported = False
+    flag_checking_supported = False
+
     def run(self) -> None:
         rule = self.test_env['rule']
 

testsuite/drivers/parser_driver.py

@@ -15,6 +15,7 @@ class RefactorDriver(BaseDriver):
     """
 
     perf_supported = True
+    flag_checking_supported = False
 
     def run(self) -> None:
         self.check_run([

testsuite/drivers/refactor_driver.py

@@ -1,3 +1,5 @@
 driver: checker
 rule_name: KP_19142
 project: prj.gpr
+control:
+  - ["XFAIL", "True", "KP detector is invalid (eng/libadalang/langkit-query-language#506)"]

testsuite/tests/checks/KP-19142/test.yaml

@@ -1,5 +1,5 @@
 separate (Test)
-procedure Prim3 (Self : T6; Other : Integer) is
+procedure Prim3 (Self : T6; Other : Integer) is  --  FLAG
 begin
    null;
 end Prim3;