Improve regex preprocessing: replace end-of-string '\Z' by '$'.

Author: fangerer

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SREModuleBuiltins.java

@@ -70,7 +70,7 @@ protected List<? extends NodeFactory<? extends PythonBuiltinNode>> getNodeFactor
     @Builtin(name = "tregex_preprocess", fixedNumOfArguments = 1)
     @GenerateNodeFactory
     abstract static class TregexPreprocessNode extends PythonUnaryBuiltinNode {
-        @CompilationFinal private Pattern commentPattern;
+        @CompilationFinal private Pattern namedCaptGroupPattern;
 
         @Specialization
         Object run(PString str) {
@@ -79,30 +79,51 @@ Object run(PString str) {
 
         @Specialization
         Object run(String str) {
-            if (commentPattern == null) {
+            if (namedCaptGroupPattern == null) {
                 CompilerDirectives.transferToInterpreterAndInvalidate();
-                commentPattern = Pattern.compile("(#[^\\]]*\n)");
+                namedCaptGroupPattern = Pattern.compile("\\?P\\<(?<GRPNAME>\\w*)\\>");
             }
             return replaceAll(str);
         }
 
-        @TruffleBoundary
+        /**
+         * replaces named capturing groups {@code ?P<name>} by {@code ?<name>}, removes comments and
+         * whitespaces if they are not in a character class, and replaces end-of-string {@code \Z}
+         * by {@code $}.
+         */
+        @TruffleBoundary(transferToInterpreterOnException = false, allowInlining = true)
         private String replaceAll(String r) {
-            Matcher matcher = commentPattern.matcher(r);
-            String res = matcher.replaceAll("");
-            StringBuilder sb = new StringBuilder();
+            Matcher matcher0 = namedCaptGroupPattern.matcher(r);
+            StringBuffer sb = new StringBuffer();
+            while (matcher0.find()) {
+                matcher0.appendReplacement(sb, "?<" + matcher0.group("GRPNAME") + ">");
+            }
+            matcher0.appendTail(sb);
+
             int charclassNestingLevel = 0;
-            for (int i = 0; i < res.length(); i++) {
-                char c = res.charAt(i);
-                if (c == '[') {
+            boolean inComment = false;
+            for (int i = 0; i < sb.length();) {
+                char c = sb.charAt(i);
+                if (c == '[' && !inComment) {
                     charclassNestingLevel++;
-                } else if (c == ']') {
+                } else if (c == ']' && !inComment) {
                     charclassNestingLevel--;
+                } else if (c == '#' && charclassNestingLevel == 0) {
+                    inComment = true;
+                } else if (c == '\n' && inComment) {
+                    inComment = false;
                 }
-                if (!Character.isWhitespace(c) || charclassNestingLevel != 0) {
-                    sb.append(res.charAt(i));
+                if (inComment || (Character.isWhitespace(c) && charclassNestingLevel == 0)) {
+                    sb.deleteCharAt(i);
+                } else {
+                    i++;
                 }
             }
+
+            for (int idx = sb.indexOf("\\Z"); idx != -1; idx = sb.indexOf("\\Z", idx + 2)) {
+                sb.replace(idx, idx + 2, "$");
+            }
+
             return sb.toString();
         }
 

graalpython/lib-graalpython/_sre.py

@@ -115,7 +115,7 @@ def __init__(self, pattern, flags, code, groups=0, groupindex=None, indexgroup=N
                 jsflags.append(jsflag)
         self.jsflags = "".join(jsflags)
 
-    def _decode_string(self, string, flags):
+    def _decode_string(self, string, flags=0):
         if isinstance(string, str):
             pattern = string
         elif isinstance(string, bytes):