[GR-14968] Fix issues around regex, pygments, and loop breaks and add tests

timfel · timfel · commit 17b132fe3da3 · 2019-04-05T07:24:17.000-07:00
PullRequest: graalpython/473
diff --git a/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java b/graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java
@@ -189,6 +189,8 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
                         subprocessArgs.add("Dgraal.TruffleCompilationExceptionsArePrinted=true");
                         subprocessArgs.add("Dgraal.TraceTruffleInlining=true");
                         subprocessArgs.add("Dgraal.TruffleTraceSplittingSummary=true");
+                        subprocessArgs.add("Dgraal.TraceTruffleTransferToInterpreter=true");
+                        subprocessArgs.add("Dgraal.TraceTruffleAssumptions=true");
                         inputArgs.remove("-debug-perf");
                     } else {
                         unrecognized.add(arg);
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_bytes.py b/graalpython/com.oracle.graal.python.test/src/tests/test_bytes.py
@@ -688,8 +688,8 @@ def test_translate_no_change(self):
 
 class ByteArrayTest(BaseLikeBytes, unittest.TestCase):
     type2test = bytearray
-    
-    def test_translate_no_change(self):
+
+    def test_translate_no_change2(self):
         b = bytearray(b'ahoj')
         self.assertIsNot(b, b.translate(None))
         self.assertIsNot(b, b.translate(None, b''))
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_for.py b/graalpython/com.oracle.graal.python.test/src/tests/test_for.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -86,3 +86,29 @@ def test_else_continue():
             continue
 
     assert ["continue", "break"] == sequence
+
+
+def test_else_break():
+    iters = 0
+    while iters < 4:
+        for i in range(iters):
+            if False:
+                break
+            iters += 1
+        else:
+            iters += 1
+            break
+    assert iters == 1, "if the for-loop doesn't break, the else should be executed and break out of the outer loop"
+
+
+def test_else_break_from_while():
+    iters = 0
+    while iters < 40:
+        while iters < 10:
+            if False:
+                break
+            iters += 1
+        else:
+            iters += 1
+            break
+    assert iters == 11, "if the while-loop doesn't break, the else should be executed and break out of the outer loop"
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SREModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SREModuleBuiltins.java
@@ -40,16 +40,17 @@
  */
 package com.oracle.graal.python.builtins.modules;
 
-import static com.oracle.graal.python.runtime.exception.PythonErrorType.RuntimeError;
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError;
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
 
 import java.io.UnsupportedEncodingException;
 import java.util.List;
 
+import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
 import com.oracle.graal.python.builtins.objects.bytes.BytesUtils;
 import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
@@ -60,20 +61,21 @@
 import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
+import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.exception.PythonErrorType;
 import com.oracle.truffle.api.CompilerAsserts;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.TruffleException;
 import com.oracle.truffle.api.dsl.Cached;
+import com.oracle.truffle.api.dsl.CachedContext;
 import com.oracle.truffle.api.dsl.Fallback;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.dsl.TypeSystemReference;
 import com.oracle.truffle.api.interop.ArityException;
 import com.oracle.truffle.api.interop.InteropLibrary;
-import com.oracle.truffle.api.interop.TruffleObject;
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.interop.UnsupportedTypeException;
 import com.oracle.truffle.api.library.CachedLibrary;
@@ -90,9 +92,16 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
     @Builtin(name = "_build_regex_engine", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class BuildRegexEngine extends PythonUnaryBuiltinNode {
-        @Specialization
+        @Specialization(guards = "!getFlag(context, WithTRegex)")
+        Object useSRE(@SuppressWarnings("unused") String code,
+                        @SuppressWarnings("unused") @CachedContext(PythonLanguage.class) PythonContext context) {
+            return PNone.NONE;
+        }
+
+        @Specialization(guards = "getFlag(context, WithTRegex)")
         @TruffleBoundary
-        Object run(String code) {
+        Object run(String code,
+                        @SuppressWarnings("unused") @CachedContext(PythonLanguage.class) PythonContext context) {
             return getContext().getEnv().parse(Source.newBuilder("regex", code, "build-regex-engine").build()).call();
         }
     }
@@ -188,8 +197,8 @@ private BytesNodes.ToBytesNode getToBytesNode() {
     @GenerateNodeFactory
     abstract static class TRegexCallCompile extends PythonTernaryBuiltinNode {
 
-        @Specialization(guards = "isForeignObject(callable)", limit = "1")
-        Object call(TruffleObject callable, Object arg1, Object arg2,
+        @Specialization(limit = "1")
+        Object call(Object callable, Object arg1, Object arg2,
                         @Cached("create()") BranchProfile syntaxError,
                         @Cached("create()") BranchProfile typeError,
                         @CachedLibrary("callable") InteropLibrary interop) {
@@ -207,21 +216,15 @@ Object call(TruffleObject callable, Object arg1, Object arg2,
                 throw e;
             }
         }
-
-        @SuppressWarnings("unused")
-        @Fallback
-        Object call(Object callable, Object arg1, Object arg2) {
-            throw raise(RuntimeError, "invalid arguments passed to tregex_call_compile");
-        }
     }
 
     @Builtin(name = "tregex_call_exec", minNumOfPositionalArgs = 3)
     @TypeSystemReference(PythonArithmeticTypes.class)
     @GenerateNodeFactory
     abstract static class TRegexCallExec extends PythonTernaryBuiltinNode {
 
-        @Specialization(guards = "isForeignObject(callable)", limit = "1")
-        Object call(TruffleObject callable, Object arg1, Number arg2,
+        @Specialization(limit = "1")
+        Object call(Object callable, Object arg1, Number arg2,
                         @Cached("create()") BranchProfile typeError,
                         @CachedLibrary("callable") InteropLibrary interop) {
             try {
@@ -231,11 +234,5 @@ Object call(TruffleObject callable, Object arg1, Number arg2,
                 throw raise(TypeError, "%s", e);
             }
         }
-
-        @SuppressWarnings("unused")
-        @Fallback
-        Object call(Object callable, Object arg1, Object arg2) {
-            throw raise(RuntimeError, "invalid arguments passed to tregex_call_exec");
-        }
     }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/AbstractBytesBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/AbstractBytesBuiltins.java
@@ -932,7 +932,7 @@ public PByteArray translate(PByteArray self, @SuppressWarnings("unused") PNone t
             return factory().createByteArray(self.getSequenceStorage().copy());
         }
 
-        @Specialization
+        @Specialization(guards = "!isNone(table)")
         public PBytes translate(PBytes self, Object table, @SuppressWarnings("unused") PNone delete) {
             byte[] bTable = getToBytesNode().execute(table);
             checkLengthOfTable(bTable);
@@ -945,7 +945,7 @@ public PBytes translate(PBytes self, Object table, @SuppressWarnings("unused") P
             return self;
         }
 
-        @Specialization
+        @Specialization(guards = "!isNone(table)")
         public PByteArray translate(PByteArray self, Object table, @SuppressWarnings("unused") PNone delete) {
             byte[] bTable = getToBytesNode().execute(table);
             checkLengthOfTable(bTable);
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/ByteArrayBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/ByteArrayBuiltins.java
@@ -633,55 +633,6 @@ private int getLength(SequenceStorage s) {
         }
     }
 
-    @Builtin(name = "translate", minNumOfPositionalArgs = 2, maxNumOfPositionalArgs = 3)
-    @GenerateNodeFactory
-    abstract static class TranslateNode extends PythonBuiltinNode {
-
-        @Child private SequenceStorageNodes.GetItemNode getSelfItemNode;
-        @Child private SequenceStorageNodes.GetItemNode getTableItemNode;
-
-        @Specialization
-        PByteArray translate(PByteArray self, PBytes table, @SuppressWarnings("unused") PNone delete) {
-            return translate(self.getSequenceStorage(), table.getSequenceStorage());
-        }
-
-        @Specialization
-        PByteArray translate(PByteArray self, PByteArray table, @SuppressWarnings("unused") PNone delete) {
-            return translate(self.getSequenceStorage(), table.getSequenceStorage());
-        }
-
-        private PByteArray translate(SequenceStorage selfStorage, SequenceStorage tableStorage) {
-            if (tableStorage.length() != 256) {
-                throw raise(ValueError, "translation table must be 256 characters long");
-            }
-            byte[] result = new byte[selfStorage.length()];
-            for (int i = 0; i < selfStorage.length(); i++) {
-                int b = getGetSelfItemNode().executeInt(selfStorage, i);
-                int t = getGetTableItemNode().executeInt(tableStorage, b);
-                assert t >= 0 && t < 256;
-                result[i] = (byte) t;
-            }
-            return factory().createByteArray(result);
-        }
-
-        private SequenceStorageNodes.GetItemNode getGetSelfItemNode() {
-            if (getSelfItemNode == null) {
-                CompilerDirectives.transferToInterpreterAndInvalidate();
-                getSelfItemNode = insert(SequenceStorageNodes.GetItemNode.create());
-            }
-            return getSelfItemNode;
-        }
-
-        private SequenceStorageNodes.GetItemNode getGetTableItemNode() {
-            if (getTableItemNode == null) {
-                CompilerDirectives.transferToInterpreterAndInvalidate();
-                getTableItemNode = insert(SequenceStorageNodes.GetItemNode.create());
-            }
-            return getTableItemNode;
-        }
-
-    }
-
     @Builtin(name = __GETITEM__, minNumOfPositionalArgs = 2)
     @GenerateNodeFactory
     abstract static class GetitemNode extends PythonBinaryBuiltinNode {
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PyUnicodeWrappers.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PyUnicodeWrappers.java
@@ -47,7 +47,6 @@
 import com.oracle.graal.python.builtins.objects.cext.DynamicObjectNativeWrapper.ToPyObjectNode;
 import com.oracle.graal.python.builtins.objects.cext.UnicodeObjectNodes.UnicodeAsWideCharNode;
 import com.oracle.graal.python.builtins.objects.str.PString;
-import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
@@ -226,12 +225,16 @@ protected Object readMember(String member,
 
         private boolean onlyAscii(String value) {
             if (asciiEncoder == null) {
-                CompilerDirectives.transferToInterpreterAndInvalidate();
-                asciiEncoder = Charset.forName("US-ASCII").newEncoder();
+                asciiEncoder = newAsciiEncoder();
             }
             return doCheck(value, asciiEncoder);
         }
 
+        @TruffleBoundary
+        private static CharsetEncoder newAsciiEncoder() {
+            return Charset.forName("US-ASCII").newEncoder();
+        }
+
         @TruffleBoundary
         private static boolean doCheck(String value, CharsetEncoder asciiEncoder) {
             return asciiEncoder.canEncode(value);
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/NodeFactory.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/NodeFactory.java
@@ -209,7 +209,11 @@ public StatementNode createContinueTarget(StatementNode child) {
     }
 
     public StatementNode createBreakTarget(StatementNode forNode) {
-        return new BreakTargetNode(forNode);
+        return new BreakTargetNode(forNode, null);
+    }
+
+    public StatementNode createBreakTarget(StatementNode forNode, StatementNode orelse) {
+        return new BreakTargetNode(forNode, orelse);
     }
 
     public YieldNode createYield(ExpressionNode right, FrameSlot returnSlot) {
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/control/BreakTargetNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/control/BreakTargetNode.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
  * Copyright (c) 2014, Regents of the University of California
  *
  * All rights reserved.
@@ -36,9 +36,11 @@ public final class BreakTargetNode extends StatementNode {
     private final BranchProfile defaultExitProfile = BranchProfile.create();
 
     @Child private StatementNode statement;
+    @Child private StatementNode orelse;
 
-    public BreakTargetNode(StatementNode statement) {
+    public BreakTargetNode(StatementNode statement, StatementNode orelse) {
         this.statement = statement;
+        this.orelse = orelse;
     }
 
     public StatementNode getStatement() {
@@ -52,6 +54,10 @@ public void executeVoid(VirtualFrame frame) {
             defaultExitProfile.enter();
         } catch (BreakException ex) {
             breakProfile.enter();
+            return;
+        }
+        if (orelse != null) {
+            orelse.executeVoid(frame);
         }
     }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonTreeTranslator.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonTreeTranslator.java
@@ -1255,15 +1255,17 @@ private PNode createWhileNode(PNode test, StatementNode body, StatementNode orel
         StatementNode whileNode = factory.createWhile(factory.toBooleanCastNode(test), wrappedBody);
         // assignSourceFromNode(node, whileNode);
 
-        if (!EmptyNode.isEmpty(orelse)) {
+        if (info.hasBreak()) {
+            if (!EmptyNode.isEmpty(orelse)) {
+                whileNode = factory.createBreakTarget(whileNode, orelse);
+            } else {
+                whileNode = factory.createBreakTarget(whileNode);
+            }
+        } else if (!EmptyNode.isEmpty(orelse)) {
             whileNode = factory.createElse(whileNode, orelse);
         }
 
-        if (info.hasBreak()) {
-            return factory.createBreakTarget(whileNode);
-        } else {
-            return whileNode;
-        }
+        return whileNode;
     }
 
     @Override
@@ -1288,15 +1290,17 @@ private StatementNode createForNode(StatementNode target, ExpressionNode iter, S
         StatementNode forNode = createForInScope(target, iter, wrappedBody);
         // assignSourceFromNode(node, forNode);
 
-        if (!EmptyNode.isEmpty(orelse)) {
+        if (info.hasBreak()) {
+            if (!EmptyNode.isEmpty(orelse)) {
+                forNode = factory.createBreakTarget(forNode, orelse);
+            } else {
+                forNode = factory.createBreakTarget(forNode);
+            }
+        } else if (!EmptyNode.isEmpty(orelse)) {
             forNode = factory.createElse(forNode, orelse);
         }
 
-        if (info.hasBreak()) {
-            return factory.createBreakTarget(forNode);
-        } else {
-            return forNode;
-        }
+        return forNode;
     }
 
     private LoopNode createForInScope(StatementNode target, ExpressionNode iterator, StatementNode body) {
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java
@@ -130,6 +130,9 @@ private PythonOptions() {
     @Option(category = OptionCategory.EXPERT, help = "This option control builtin _thread module support") //
     public static final OptionKey<Boolean> WithThread = new OptionKey<>(false);
 
+    @Option(category = OptionCategory.EXPERT, help = "Use the optimized TRegex engine and call the CPython sre engine only as a fallback. Default true") //
+    public static final OptionKey<Boolean> WithTRegex = new OptionKey<>(true);
+
     @Option(category = OptionCategory.EXPERT, help = "Switch on/off using lazy strings for performance reasons. Default true.") //
     public static final OptionKey<Boolean> LazyStrings = new OptionKey<>(true);
 
diff --git a/graalpython/lib-graalpython/_sre.py b/graalpython/lib-graalpython/_sre.py

Original file line number	Diff line number	Diff line change
`@@ -209,7 +209,11 @@ public StatementNode createContinueTarget(StatementNode child) {`
`209`	`209`	`}`
`210`	`210`
`211`	`211`	`public StatementNode createBreakTarget(StatementNode forNode) {`
`212`		`- return new BreakTargetNode(forNode);`
	`212`	`+ return new BreakTargetNode(forNode, null);`
	`213`	`+ }`
	`214`	`+`
	`215`	`+ public StatementNode createBreakTarget(StatementNode forNode, StatementNode orelse) {`
	`216`	`+ return new BreakTargetNode(forNode, orelse);`
`213`	`217`	`}`
`214`	`218`
`215`	`219`	`public YieldNode createYield(ExpressionNode right, FrameSlot returnSlot) {`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`/*`
`2`		`- * Copyright (c) 2017, 2018, Oracle and/or its affiliates.`
	`2`	`+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates.`
`3`	`3`	`* Copyright (c) 2014, Regents of the University of California`
`4`	`4`	`*`
`5`	`5`	`* All rights reserved.`
`@@ -36,9 +36,11 @@ public final class BreakTargetNode extends StatementNode {`
`36`	`36`	`private final BranchProfile defaultExitProfile = BranchProfile.create();`
`37`	`37`
`38`	`38`	`@Child private StatementNode statement;`
	`39`	`+ @Child private StatementNode orelse;`
`39`	`40`
`40`		`- public BreakTargetNode(StatementNode statement) {`
	`41`	`+ public BreakTargetNode(StatementNode statement, StatementNode orelse) {`
`41`	`42`	`this.statement = statement;`
	`43`	`+ this.orelse = orelse;`
`42`	`44`	`}`
`43`	`45`
`44`	`46`	`public StatementNode getStatement() {`
`@@ -52,6 +54,10 @@ public void executeVoid(VirtualFrame frame) {`
`52`	`54`	`defaultExitProfile.enter();`
`53`	`55`	`} catch (BreakException ex) {`
`54`	`56`	`breakProfile.enter();`
	`57`	`+ return;`
	`58`	`+ }`
	`59`	`+ if (orelse != null) {`
	`60`	`+ orelse.executeVoid(frame);`
`55`	`61`	`}`
`56`	`62`	`}`
`57`	`63`	`}`