[GR-46597] Make future annotations inherited by exec

PullRequest: graalpython/2893

Author: Adam Hrbac

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/FutureFeature.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -40,7 +40,35 @@
  */
 package com.oracle.graal.python.pegparser;
 
+import java.util.EnumSet;
+
 public enum FutureFeature {
-    ANNOTATIONS,
-    BARRY_AS_BDFL
+    ANNOTATIONS(0x1000000),
+    BARRY_AS_BDFL(0x400000);
+
+    public final int flagValue;
+    public static final int ALL_FLAGS;
+
+    private static final FutureFeature[] VALUES = FutureFeature.values();
+    static {
+        int flags = 0;
+        for (FutureFeature feat : VALUES) {
+            flags |= feat.flagValue;
+        }
+        ALL_FLAGS = flags;
+    }
+
+    FutureFeature(int flagValue) {
+        this.flagValue = flagValue;
+    }
+
+    public static EnumSet<FutureFeature> fromFlags(int flags) {
+        EnumSet<FutureFeature> set = EnumSet.noneOf(FutureFeature.class);
+        for (FutureFeature feat : VALUES) {
+            if ((feat.flagValue & flags) != 0) {
+                set.add(feat);
+            }
+        }
+        return set;
+    }
 }

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/basic/FutureAnnotationTests.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.test.basic;
+
+import static com.oracle.graal.python.test.PythonTests.assertPrints;
+
+import org.junit.Test;
+
+public class FutureAnnotationTests {
+    @Test
+    public void withoutEvaluates() {
+        assertPrints("hello\n", "def f() -> print('hello'): pass");
+    }
+
+    @Test
+    public void withDoesNotEvaluate() {
+        assertPrints("", "from __future__ import annotations\ndef f() -> print('hello'): pass");
+    }
+
+    @Test
+    public void worksInExec() {
+        assertPrints("hello\n", "exec('def f() -> print(\\'hello\\'): pass')");
+        assertPrints("", "exec('from __future__ import annotations\\ndef f() -> print(\\'hello\\'): pass')");
+    }
+
+    @Test
+    public void execInherits() {
+        assertPrints("", "from __future__ import annotations\nexec('def f() -> print(\\'hello\\'): pass')");
+    }
+
+}

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/compiler/CompilerTests.java

@@ -58,6 +58,7 @@
 import com.oracle.graal.python.compiler.CompilationUnit;
 import com.oracle.graal.python.compiler.Compiler;
 import com.oracle.graal.python.pegparser.ErrorCallback;
+import com.oracle.graal.python.pegparser.FutureFeature;
 import com.oracle.graal.python.pegparser.InputType;
 import com.oracle.graal.python.pegparser.Parser;
 import com.oracle.graal.python.pegparser.sst.ModTy;
@@ -1096,7 +1097,7 @@ private static CodeUnit assemble(String src, InputType type) {
         Parser parser = Compiler.createParser(src, errorCallback, type, false);
         ModTy result = (ModTy) parser.parse();
         Compiler compiler = new Compiler(errorCallback);
-        CompilationUnit cu = compiler.compile(result, EnumSet.noneOf(Compiler.Flags.class), 2);
+        CompilationUnit cu = compiler.compile(result, EnumSet.noneOf(Compiler.Flags.class), 2, EnumSet.noneOf(FutureFeature.class));
         return cu.assemble();
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -35,8 +35,10 @@
 
 import java.io.IOException;
 import java.nio.file.InvalidPathException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
@@ -74,6 +76,7 @@
 import com.oracle.graal.python.nodes.exception.TopLevelExceptionHandler;
 import com.oracle.graal.python.nodes.frame.GetFrameLocalsNode;
 import com.oracle.graal.python.nodes.frame.MaterializeFrameNode;
+import com.oracle.graal.python.pegparser.FutureFeature;
 import com.oracle.graal.python.pegparser.InputType;
 import com.oracle.graal.python.pegparser.NodeFactory;
 import com.oracle.graal.python.pegparser.Parser;
@@ -132,8 +135,11 @@
                 implementationName = PythonLanguage.IMPLEMENTATION_NAME, //
                 version = PythonLanguage.VERSION, //
                 characterMimeTypes = {PythonLanguage.MIME_TYPE,
-                                PythonLanguage.MIME_TYPE_COMPILE0, PythonLanguage.MIME_TYPE_COMPILE1, PythonLanguage.MIME_TYPE_COMPILE2,
-                                PythonLanguage.MIME_TYPE_EVAL0, PythonLanguage.MIME_TYPE_EVAL1, PythonLanguage.MIME_TYPE_EVAL2}, //
+                                "text/x-python-\0\u0000-eval", "text/x-python-\0\u0000-compile", "text/x-python-\1\u0000-eval", "text/x-python-\1\u0000-compile", "text/x-python-\2\u0000-eval",
+                                "text/x-python-\2\u0000-compile", "text/x-python-\0\u0100-eval", "text/x-python-\0\u0100-compile", "text/x-python-\1\u0100-eval", "text/x-python-\1\u0100-compile",
+                                "text/x-python-\2\u0100-eval", "text/x-python-\2\u0100-compile", "text/x-python-\0\u0040-eval", "text/x-python-\0\u0040-compile", "text/x-python-\1\u0040-eval",
+                                "text/x-python-\1\u0040-compile", "text/x-python-\2\u0040-eval", "text/x-python-\2\u0040-compile", "text/x-python-\0\u0140-eval", "text/x-python-\0\u0140-compile",
+                                "text/x-python-\1\u0140-eval", "text/x-python-\1\u0140-compile", "text/x-python-\2\u0140-eval", "text/x-python-\2\u0140-compile"}, //
                 byteMimeTypes = {PythonLanguage.MIME_TYPE_BYTECODE}, //
                 defaultMimeType = PythonLanguage.MIME_TYPE, //
                 dependentLanguages = {"nfi", "llvm"}, //
@@ -206,14 +212,49 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     public static final int API_VERSION = 1013;
 
     public static final String MIME_TYPE = "text/x-python";
-    static final String MIME_TYPE_COMPILE0 = "text/x-python-compile0";
-    static final String MIME_TYPE_COMPILE1 = "text/x-python-compile1";
-    static final String MIME_TYPE_COMPILE2 = "text/x-python-compile2";
-    static final String[] MIME_TYPE_COMPILE = {PythonLanguage.MIME_TYPE_COMPILE0, PythonLanguage.MIME_TYPE_COMPILE1, PythonLanguage.MIME_TYPE_COMPILE2};
-    static final String[] MIME_TYPE_EVAL = {PythonLanguage.MIME_TYPE_EVAL0, PythonLanguage.MIME_TYPE_EVAL1, PythonLanguage.MIME_TYPE_EVAL2};
-    static final String MIME_TYPE_EVAL0 = "text/x-python-eval0";
-    static final String MIME_TYPE_EVAL1 = "text/x-python-eval1";
-    static final String MIME_TYPE_EVAL2 = "text/x-python-eval2";
+
+    // the syntax for mime types is as follows
+    // <mime> ::= "text/x-python-" <optlevel> <flags> "-" kind
+    // <kind> ::= "compile" | "eval"
+    // <optlevel> ::= "\0" | "\1" | "\2"
+    // <flags> ::= "\u0040" | "\u0100" | "\u0140" | "\u0000"
+    // where 0100 implies annotations, and 0040 implies barry_as_flufl
+    static final String MIME_PREFIX = MIME_TYPE + "-";
+    static final int OPT_FLAGS_LEN = 2; // 1 char is optlevel, 1 char is flags
+    static final String MIME_KIND_COMPILE = "compile";
+    static final String MIME_KIND_EVAL = "eval";
+    // Since flags are greater than the highest unicode codepoint, we shift them into more
+    // reasonable values in the mime type. 4 hex digits
+    static final int MIME_FLAG_SHIFTBY = 4 * 4;
+    // a dash follows after the opt flag pair
+    static final int MIME_KIND_START = MIME_PREFIX.length() + OPT_FLAGS_LEN + 1;
+
+    private static boolean mimeTypesComplete(ArrayList<String> mimeJavaStrings) {
+        ArrayList<String> mimeTypes = new ArrayList<>();
+        FutureFeature[] all = FutureFeature.values();
+        for (int flagset = 0; flagset < (1 << all.length); ++flagset) {
+            int flags = 0;
+            for (int i = 0; i < all.length; ++i) {
+                if ((flagset & (1 << i)) != 0) {
+                    flags |= all[i].flagValue;
+                }
+            }
+            for (int opt = 0; opt <= 2; opt++) {
+                for (String typ : new String[]{MIME_KIND_EVAL, MIME_KIND_COMPILE}) {
+                    mimeTypes.add(MIME_PREFIX + optFlagsToMime(opt, flags) + "-" + typ);
+                    mimeJavaStrings.add(String.format("\"%s\\%d\\u%04x-%s\"", MIME_PREFIX, opt, flags >> MIME_FLAG_SHIFTBY, typ));
+                }
+            }
+        }
+        HashSet<String> currentMimeTypes = new HashSet<>(List.of(PythonLanguage.class.getAnnotation(Registration.class).characterMimeTypes()));
+        return currentMimeTypes.containsAll(mimeTypes);
+    }
+
+    static {
+        ArrayList<String> mimeJavaStrings = new ArrayList<>();
+        assert mimeTypesComplete(mimeJavaStrings) : "Expected all of {" + String.join(", ", mimeJavaStrings) + "} in the PythonLanguage characterMimeTypes";
+    }
+
     public static final String MIME_TYPE_BYTECODE = "application/x-python-bytecode";
 
     public static final TruffleString[] T_DEFAULT_PYTHON_EXTENSIONS = new TruffleString[]{T_PY_EXTENSION, tsLiteral(".pyc")};
@@ -388,24 +429,26 @@ protected void initializeContext(PythonContext context) {
         context.initialize();
     }
 
-    public static String getCompileMimeType(int optimize) {
-        if (optimize <= 0) {
-            return MIME_TYPE_COMPILE0;
-        } else if (optimize == 1) {
-            return MIME_TYPE_COMPILE1;
-        } else {
-            return MIME_TYPE_COMPILE2;
+    private static String optFlagsToMime(int optimize, int flags) {
+        if (optimize < 0) {
+            optimize = 0;
+        } else if (optimize > 2) {
+            optimize = 2;
         }
+        String optField = new String(new byte[]{(byte) optimize});
+        String flagField = new String(new int[]{(flags & FutureFeature.ALL_FLAGS) >> MIME_FLAG_SHIFTBY}, 0, 1);
+        assert flagField.length() == 1 : "flags in mime type ended up a surrogate";
+        return optField + flagField;
     }
 
-    public static String getEvalMimeType(int optimize) {
-        if (optimize <= 0) {
-            return MIME_TYPE_EVAL0;
-        } else if (optimize == 1) {
-            return MIME_TYPE_EVAL1;
-        } else {
-            return MIME_TYPE_EVAL2;
-        }
+    public static String getCompileMimeType(int optimize, int flags) {
+        String optFlags = optFlagsToMime(optimize, flags);
+        return MIME_PREFIX + optFlags + "-compile";
+    }
+
+    public static String getEvalMimeType(int optimize, int flags) {
+        String optFlags = optFlagsToMime(optimize, flags);
+        return MIME_PREFIX + optFlags + "-eval";
     }
 
     @Override
@@ -417,7 +460,7 @@ protected CallTarget parse(ParsingRequest request) {
                 throw new IllegalStateException("parse with arguments not allowed for interactive sources");
             }
             InputType inputType = source.isInteractive() ? InputType.SINGLE : InputType.FILE;
-            return parse(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames());
+            return parse(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames(), EnumSet.noneOf(FutureFeature.class));
         }
         if (!request.getArgumentNames().isEmpty()) {
             throw new IllegalStateException("parse with arguments is only allowed for " + MIME_TYPE + " mime type");
@@ -452,19 +495,28 @@ protected CallTarget parse(ParsingRequest request) {
             PBytecodeRootNode rootNode = PBytecodeRootNode.create(this, code, source);
             return PythonUtils.getOrCreateCallTarget(rootNode);
         }
-        for (int optimize = 0; optimize < MIME_TYPE_EVAL.length; optimize++) {
-            if (MIME_TYPE_EVAL[optimize].equals(source.getMimeType())) {
-                assert !source.isInteractive();
-                return parse(context, source, InputType.EVAL, false, optimize, false, null);
-            }
+
+        String mime = source.getMimeType();
+        String prefix = mime.substring(0, MIME_PREFIX.length());
+        if (!prefix.equals(MIME_PREFIX)) {
+            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + mime);
         }
-        for (int optimize = 0; optimize < MIME_TYPE_COMPILE.length; optimize++) {
-            if (MIME_TYPE_COMPILE[optimize].equals(source.getMimeType())) {
-                assert !source.isInteractive();
-                return parse(context, source, InputType.FILE, false, optimize, false, null);
-            }
+        String kind = mime.substring(MIME_KIND_START);
+        InputType type;
+        if (kind.equals(MIME_KIND_COMPILE)) {
+            type = InputType.FILE;
+        } else if (kind.equals(MIME_KIND_EVAL)) {
+            type = InputType.EVAL;
+        } else {
+            throw CompilerDirectives.shouldNotReachHere("unknown compilation kind: " + kind + " from mime type: " + mime);
+        }
+        int optimize = mime.codePointAt(MIME_PREFIX.length());
+        int flags = mime.codePointAt(MIME_PREFIX.length() + 1) << MIME_FLAG_SHIFTBY;
+        if (0 > optimize || optimize > 2 || (flags & ~FutureFeature.ALL_FLAGS) != 0) {
+            throw CompilerDirectives.shouldNotReachHere("Invalid value for optlevel or flags: " + optimize + "," + flags + " from mime type: " + mime);
         }
-        throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
+        assert !source.isInteractive();
+        return parse(context, source, type, false, optimize, false, null, FutureFeature.fromFlags(flags));
     }
 
     private static Source tryLoadSource(PythonContext context, CodeUnit code, boolean internal, String path) {
@@ -475,13 +527,14 @@ private static Source tryLoadSource(PythonContext context, CodeUnit code, boolea
         }
     }
 
-    public RootCallTarget parse(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames) {
+    public RootCallTarget parse(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames,
+                    EnumSet<FutureFeature> futureFeatures) {
         RaisePythonExceptionErrorCallback errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
         try {
             Parser parser = Compiler.createParser(source.getCharacters().toString(), errorCb, type, interactiveTerminal);
             ModTy mod = (ModTy) parser.parse();
             assert mod != null;
-            return compileForBytecodeInterpreter(context, mod, source, topLevel, optimize, argumentNames, errorCb);
+            return compileForBytecodeInterpreter(context, mod, source, topLevel, optimize, argumentNames, errorCb, futureFeatures);
         } catch (PException e) {
             if (topLevel) {
                 PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e)).call();
@@ -492,7 +545,13 @@ public RootCallTarget parse(PythonContext context, Source source, InputType type
 
     @TruffleBoundary
     public RootCallTarget compileForBytecodeInterpreter(PythonContext context, ModTy mod, Source source, boolean topLevel, int optimize, List<String> argumentNames,
-                    RaisePythonExceptionErrorCallback errorCallback) {
+                    RaisePythonExceptionErrorCallback errorCallback, int flags) {
+        return compileForBytecodeInterpreter(context, mod, source, topLevel, optimize, argumentNames, errorCallback, FutureFeature.fromFlags(flags));
+    }
+
+    @TruffleBoundary
+    public RootCallTarget compileForBytecodeInterpreter(PythonContext context, ModTy mod, Source source, boolean topLevel, int optimize, List<String> argumentNames,
+                    RaisePythonExceptionErrorCallback errorCallback, EnumSet<FutureFeature> futureFeatures) {
         RaisePythonExceptionErrorCallback errorCb = errorCallback;
         if (errorCb == null) {
             errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
@@ -503,7 +562,7 @@ public RootCallTarget compileForBytecodeInterpreter(PythonContext context, ModTy
             if (hasArguments) {
                 mod = transformASTForExecutionWithArguments(argumentNames, mod);
             }
-            CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize);
+            CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureFeatures);
             CodeUnit co = cu.assemble();
             RootNode rootNode = PBytecodeRootNode.create(this, co, source, errorCb);
             if (topLevel) {
@@ -588,7 +647,7 @@ public String getName() {
     @Override
     public ExecutableNode parse(InlineParsingRequest request) {
         PythonContext context = PythonContext.get(null);
-        RootCallTarget callTarget = parse(context, request.getSource(), InputType.EVAL, false, 0, false, null);
+        RootCallTarget callTarget = parse(context, request.getSource(), InputType.EVAL, false, 0, false, null, EnumSet.noneOf(FutureFeature.class));
         return new ExecutableNode(this) {
             @Child private GilNode gilNode = GilNode.create();
             @Child private GenericInvokeNode invokeNode = GenericInvokeNode.create();