Implement the --error-filter-patterns option

Author: theosotr

hephaestus.py

@@ -371,7 +371,8 @@ def check_oracle(dirname, oracles):
     produce the expected results (and the reason why).
     """
     filename = os.path.join(dirname, 'src')
-    compiler = COMPILERS[cli_args.language](filename)
+    filter_patterns = utils.path2set(cli_args.error_filter_patterns)
+    compiler = COMPILERS[cli_args.language](filename, filter_patterns)
     command_args = compiler.get_compiler_cmd()
     # At this point, we run the compiler
     _, err = run_command(command_args)
@@ -382,7 +383,7 @@ def check_oracle(dirname, oracles):
 
     # Analyze the compiler output and check whether there are programs
     # that the compiler did not manage to compile.
-    failed = compiler.analyze_compiler_output(err)
+    failed, _ = compiler.analyze_compiler_output(err)
     if compiler.crash_msg:
         # We just found a compiler crash.
         shutil.rmtree(dirname)

src/args.py

@@ -161,6 +161,13 @@
     action="store_true",
     help="Disable parameterized functions"
 )
+parser.add_argument(
+    "--error-filter-patterns",
+    default=None,
+    type=str,
+    help=("A file containing regular expressions for filtering compiler error "
+          "messages")
+)
 
 
 args = parser.parse_args()

src/compilers/base.py

@@ -1,6 +1,15 @@
+from collections import defaultdict
+import re
+
+
 class BaseCompiler():
-    def __init__(self, input_name):
+    ERROR_REGEX = None
+    CRASH_REGEX = None
+
+    def __init__(self, input_name, filter_patterns=None):
         self.input_name = input_name
+        self.filter_patterns = filter_patterns or []
+        self.crash_msg = None
 
     @classmethod
     def get_compiler_version(cls):
@@ -9,6 +18,25 @@ def get_compiler_version(cls):
     def get_compiler_cmd(self):
         raise NotImplementedError('get_compiler_cmd() must be implemented')
 
-    def analyze_compiler_output(cls, output):
-        raise NotImplementedError(
-            'analyze_compiler_output() must be implemented')
+    def get_filename(self, match):
+        raise NotImplementedError('get_filename() must be implemented')
+
+    def get_error_msg(self, match):
+        raise NotImplementedError('get_error_msg() must be implemented')
+
+    def analyze_compiler_output(self, output):
+        failed = defaultdict(list)
+        filtered_output = output
+        for p in self.filter_patterns:
+            filtered_output = re.sub(p, '', output)
+        matches = re.findall(self.ERROR_REGEX, filtered_output)
+        for match in matches:
+            filename = self.get_filename(match)
+            error_msg = self.get_error_msg(match)
+            failed[filename].append(error_msg)
+
+        crash_match = re.search(self.CRASH_REGEX, filtered_output)
+        if crash_match and not matches:
+            self.crash_msg = output
+            return None, matches
+        return failed, matches

src/compilers/groovy.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 import re
 import os
 
@@ -13,10 +12,9 @@ class GroovyCompiler(BaseCompiler):
 
     STACKOVERFLOW_REGEX = re.compile(r'(.*java.lang.StackOverflowError)(.*)')
 
-    def __init__(self, input_name):
+    def __init__(self, input_name, filter_patterns=None):
         input_name = os.path.join(input_name, '*', '*.groovy')
-        super().__init__(input_name)
-        self.crash_msg = None
+        super().__init__(input_name, filter_patterns)
 
     @classmethod
     def get_compiler_version(cls):
@@ -25,23 +23,17 @@ def get_compiler_version(cls):
     def get_compiler_cmd(self):
         return ['groovyc', '--compile-static', self.input_name]
 
-    def analyze_compiler_output(self, output):
-        self.crashed = None
-        failed = defaultdict(list)
-        matches = re.findall(self.ERROR_REGEX, output)
-        for match in matches:
-            filename = match[0]
-            error_msg = match[1]
-            failed[filename].append(error_msg)
-
-        crash_match = re.search(self.CRASH_REGEX, output)
-        if crash_match and not matches:
-            self.crash_msg = output
-            return None
+    def get_filename(self, match):
+        return match[0]
 
+    def get_error_msg(self, match):
+        return match[1]
+
+    def analyze_compiler_output(self, output):
+        failed, matches = super().analyze_compiler_output(output)
         stack_overflow = re.search(self.STACKOVERFLOW_REGEX, output)
         if stack_overflow and not matches:
             self.crash_msg = output
-            return None
+            return None, matches
 
-        return failed
+        return failed, matches

src/compilers/java.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 import re
 import os
 
@@ -12,10 +11,9 @@ class JavaCompiler(BaseCompiler):
 
     CRASH_REGEX = re.compile(r'(java\.lang.*)\n(.*)')
 
-    def __init__(self, input_name):
+    def __init__(self, input_name, filter_patterns=None):
         input_name = os.path.join(input_name, '*', '*.java')
-        super().__init__(input_name)
-        self.crash_msg = None
+        super().__init__(input_name, filter_patterns)
 
     @classmethod
     def get_compiler_version(cls):
@@ -24,17 +22,8 @@ def get_compiler_version(cls):
     def get_compiler_cmd(self):
         return ['javac', '-nowarn', self.input_name]
 
-    def analyze_compiler_output(self, output):
-        self.crashed = None
-        failed = defaultdict(list)
-        matches = re.findall(self.ERROR_REGEX, output)
-        for match in matches:
-            filename = match[0]
-            error_msg = match[1]
-            failed[filename].append(error_msg)
-
-        crash_match = re.search(self.CRASH_REGEX, output)
-        if crash_match and not matches:
-            self.crash_msg = output
-            return None
-        return failed
+    def get_filename(self, match):
+        return match[0]
+
+    def get_error_msg(self, match):
+        return match[1]

src/compilers/kotlin.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 import re
 
 from src.compilers.base import BaseCompiler
@@ -12,9 +11,8 @@ class KotlinCompiler(BaseCompiler):
         re.MULTILINE
     )
 
-    def __init__(self, input_name):
-        super().__init__(input_name)
-        self.crash_msg = None
+    def __init__(self, input_name, filter_patterns=None):
+        super().__init__(input_name, filter_patterns)
 
     @classmethod
     def get_compiler_version(cls):
@@ -24,17 +22,8 @@ def get_compiler_cmd(self):
         return ['kotlinc', self.input_name, '-include-runtime', '-d',
                 'program.jar']
 
-    def analyze_compiler_output(self, output):
-        self.crashed = None
-        failed = defaultdict(list)
-        matches = re.findall(self.ERROR_REGEX, output)
-        for match in matches:
-            filename = match[0]
-            error_msg = match[1]
-            failed[filename].append(error_msg)
-
-        match = re.search(self.CRASH_REGEX, output)
-        if match and not matches:
-            self.crash_msg = ':'.join(match.groups())
-            return None
-        return failed
+    def get_filename(self, match):
+        return match[0]
+
+    def get_error_msg(self, match):
+        return match[1]

src/utils.py

@@ -94,9 +94,7 @@ def save_text(path, text):
         out.write(text)
 
 
-def get_reserved_words(resource_path, language):
-    filename = "{}_keywords".format(language)
-    path = os.path.join(resource_path, filename)
+def path2set(path):
     if os.path.isfile(path):
         with open(path, 'r') as f:
             return {
@@ -107,6 +105,12 @@ def get_reserved_words(resource_path, language):
         return set()
 
 
+def get_reserved_words(resource_path, language):
+    filename = "{}_keywords".format(language)
+    path = os.path.join(resource_path, filename)
+    return path2set(path)
+
+
 class RandomUtils():
 
     resource_path = os.path.join(os.path.split(__file__)[0], "resources")