Merge branch 'tim/sklearn-linreg' of ssh://ol-bitbucket.us.oracle.com:7999/g/graalpython into tim/sklearn-linreg

Author: timfel

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SREModuleBuiltins.java

@@ -38,18 +38,67 @@
  */
 package com.oracle.graal.python.builtins.modules;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
+import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
+import com.oracle.graal.python.runtime.exception.PythonErrorType;
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
+import com.oracle.truffle.api.dsl.Fallback;
+import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
-
-import java.util.ArrayList;
-import java.util.List;
+import com.oracle.truffle.api.dsl.Specialization;
 
 @CoreFunctions(defineModule = "_sre")
 public class SREModuleBuiltins extends PythonBuiltins {
     @Override
     protected List<? extends NodeFactory<? extends PythonBuiltinNode>> getNodeFactories() {
         return new ArrayList<>();
     }
+
+    /**
+     * Called from C when they actually want a {@code const char*} for a Python string
+     */
+    @Builtin(name = "tregex_preprocess", fixedNumOfArguments = 1)
+    @GenerateNodeFactory
+    abstract static class TregexPreprocessNode extends PythonUnaryBuiltinNode {
+        @CompilationFinal private Pattern pattern;
+
+        @Specialization
+        Object run(PString str) {
+            return run(str.getValue());
+        }
+
+        @Specialization
+        Object run(String str) {
+            str.replaceAll("[^\\[]?#[^\\]]*\n", "");
+            if (pattern == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                pattern = Pattern.compile("(?<CMT>#[^\\]]*\n)");
+            }
+            return replaceAll(str);
+        }
+
+        @TruffleBoundary
+        private String replaceAll(String r) {
+            Matcher matcher = pattern.matcher(r);
+            return matcher.replaceAll("");
+        }
+
+        @Fallback
+        Object run(Object o) {
+            throw raise(PythonErrorType.TypeError, "expected string, not %p", o);
+        }
+
+    }
+
 }

graalpython/lib-graalpython/_sre.py

@@ -103,7 +103,7 @@ def lastindex(self):
 
 class SRE_Pattern():
     def __init__(self, pattern, flags, code, groups=0, groupindex=None, indexgroup=None):
-        self.pattern = self._decode_string(pattern)
+        self.pattern = self._decode_string(pattern, flags)
         self.flags = flags
         self.code = code
         self.num_groups = groups
@@ -115,15 +115,22 @@ def __init__(self, pattern, flags, code, groups=0, groupindex=None, indexgroup=N
                 jsflags.append(jsflag)
         self.jsflags = "".join(jsflags)
 
-    def _decode_string(self, string):
+    def _decode_string(self, string, flags):
         if isinstance(string, str):
             pattern = string
         elif isinstance(string, bytes):
             pattern = string.decode()
         else:
             raise TypeError("invalid search pattern {!r}".format(string))
         # TODO: fix this in the regex engine
-        return pattern.replace(r'\"', '"').replace(r"\'", "'")
+        pattern = pattern.replace(r'\"', '"').replace(r"\'", "'")
+
+        # TODO: that's not nearly complete but should be sufficient for now
+        from sre_compile import SRE_FLAG_VERBOSE
+        if flags & SRE_FLAG_VERBOSE:
+            pattern = _sre.tregex_preprocess(pattern)
+        return pattern
+
 
     def __repr__(self):
         flags = self.flags