Change mime format and support all future feats

Author: Adam Hrbac

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/FutureFeature.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -40,13 +40,35 @@
  */
 package com.oracle.graal.python.pegparser;
 
+import java.util.EnumSet;
+
 public enum FutureFeature {
-    ANNOTATIONS(0x1000000), // these values are duplicated in BuiltinFunctions.Compiler
+    ANNOTATIONS(0x1000000),
     BARRY_AS_BDFL(0x400000);
 
     public final int flagValue;
+    public static final int ALL_FLAGS;
+
+    private static final FutureFeature[] VALUES = FutureFeature.values();
+    static {
+        int flags = 0;
+        for (FutureFeature feat : VALUES) {
+            flags |= feat.flagValue;
+        }
+        ALL_FLAGS = flags;
+    }
 
     FutureFeature(int flagValue) {
         this.flagValue = flagValue;
     }
+
+    public static EnumSet<FutureFeature> fromFlags(int flags) {
+        EnumSet<FutureFeature> set = EnumSet.noneOf(FutureFeature.class);
+        for (FutureFeature feat : VALUES) {
+            if ((feat.flagValue & flags) != 0) {
+                set.add(feat);
+            }
+        }
+        return set;
+    }
 }

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/basic/FutureAnnotationTests.java

@@ -1,3 +1,43 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
 package com.oracle.graal.python.test.basic;
 
 import static com.oracle.graal.python.test.PythonTests.assertPrints;

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/compiler/CompilerTests.java

@@ -58,6 +58,7 @@
 import com.oracle.graal.python.compiler.CompilationUnit;
 import com.oracle.graal.python.compiler.Compiler;
 import com.oracle.graal.python.pegparser.ErrorCallback;
+import com.oracle.graal.python.pegparser.FutureFeature;
 import com.oracle.graal.python.pegparser.InputType;
 import com.oracle.graal.python.pegparser.Parser;
 import com.oracle.graal.python.pegparser.sst.ModTy;
@@ -1096,7 +1097,7 @@ private static CodeUnit assemble(String src, InputType type) {
         Parser parser = Compiler.createParser(src, errorCallback, type, false);
         ModTy result = (ModTy) parser.parse();
         Compiler compiler = new Compiler(errorCallback);
-        CompilationUnit cu = compiler.compile(result, EnumSet.noneOf(Compiler.Flags.class), 2, false);
+        CompilationUnit cu = compiler.compile(result, EnumSet.noneOf(Compiler.Flags.class), 2, EnumSet.noneOf(FutureFeature.class));
         return cu.assemble();
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -35,8 +35,10 @@
 
 import java.io.IOException;
 import java.nio.file.InvalidPathException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
@@ -74,6 +76,7 @@
 import com.oracle.graal.python.nodes.exception.TopLevelExceptionHandler;
 import com.oracle.graal.python.nodes.frame.GetFrameLocalsNode;
 import com.oracle.graal.python.nodes.frame.MaterializeFrameNode;
+import com.oracle.graal.python.pegparser.FutureFeature;
 import com.oracle.graal.python.pegparser.InputType;
 import com.oracle.graal.python.pegparser.NodeFactory;
 import com.oracle.graal.python.pegparser.Parser;
@@ -131,10 +134,12 @@
                 name = PythonLanguage.NAME, //
                 implementationName = PythonLanguage.IMPLEMENTATION_NAME, //
                 version = PythonLanguage.VERSION, //
-                characterMimeTypes = {PythonLanguage.MIME_TYPE, "text/x-python-eval-0-0", "text/x-python-eval-0-1", "text/x-python-eval-1-0", "text/x-python-eval-1-1", "text/x-python-eval-2-0",
-                                "text/x-python-eval-2-1", "text/x-python-compile-0-0", "text/x-python-compile-0-1", "text/x-python-compile-1-0", "text/x-python-compile-1-1",
-                                "text/x-python-compile-2-0",
-                                "text/x-python-compile-2-1"}, //
+                characterMimeTypes = {PythonLanguage.MIME_TYPE,
+                                "text/x-python-\0\u0000-eval", "text/x-python-\0\u0000-compile", "text/x-python-\1\u0000-eval", "text/x-python-\1\u0000-compile", "text/x-python-\2\u0000-eval",
+                                "text/x-python-\2\u0000-compile", "text/x-python-\0\u0100-eval", "text/x-python-\0\u0100-compile", "text/x-python-\1\u0100-eval", "text/x-python-\1\u0100-compile",
+                                "text/x-python-\2\u0100-eval", "text/x-python-\2\u0100-compile", "text/x-python-\0\u0040-eval", "text/x-python-\0\u0040-compile", "text/x-python-\1\u0040-eval",
+                                "text/x-python-\1\u0040-compile", "text/x-python-\2\u0040-eval", "text/x-python-\2\u0040-compile", "text/x-python-\0\u0140-eval", "text/x-python-\0\u0140-compile",
+                                "text/x-python-\1\u0140-eval", "text/x-python-\1\u0140-compile", "text/x-python-\2\u0140-eval", "text/x-python-\2\u0140-compile"}, //
                 byteMimeTypes = {PythonLanguage.MIME_TYPE_BYTECODE}, //
                 defaultMimeType = PythonLanguage.MIME_TYPE, //
                 dependentLanguages = {"nfi", "llvm"}, //
@@ -209,17 +214,44 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     public static final String MIME_TYPE = "text/x-python";
 
     // the syntax for mime types is as follows
-    // <mime> ::= "text/x-python-" <kind> "-" <optlevel> "-" <future_annotations>
+    // <mime> ::= "text/x-python-" <optlevel> <flags> "-" kind
     // <kind> ::= "compile" | "eval"
-    // <optlevel> ::= "0" | "1" | "2"
-    // <future_annotations> ::= "0" | "1"
+    // <optlevel> ::= "\0" | "\1" | "\2"
+    // <flags> ::= "\u0040" | "\u0100" | "\u0140" | "\u0000"
+    // where 0100 implies annotations, and 0040 implies barry_as_flufl
     static final String MIME_PREFIX = MIME_TYPE + "-";
+    static final int OPT_FLAGS_LEN = 2; // 1 char is optlevel, 1 char is flags
     static final String MIME_KIND_COMPILE = "compile";
     static final String MIME_KIND_EVAL = "eval";
-    static final int MIME_KIND_IDX = 0;
-    static final int MIME_OPTLEVEL_IDX = 1;
-    static final int MIME_FUTURE_ANNOTATIONS_IDX = 2;
-    static final int MIME_FIELD_COUNT = 3;
+    // Since flags are greater than the highest unicode codepoint, we shift them into more
+    // reasonable values in the mime type. 4 hex digits
+    static final int MIME_FLAG_SHIFTBY = 4 * 4;
+    // a dash follows after the opt flag pair
+    static final int MIME_KIND_START = MIME_PREFIX.length() + OPT_FLAGS_LEN + 1;
+
+    static {
+        ArrayList<String> mimeTypes = new ArrayList<>();
+        ArrayList<String> mimeJavaStrings = new ArrayList<>();
+        FutureFeature[] all = FutureFeature.values();
+        for (int flagset = 0; flagset < (1 << all.length); ++flagset) {
+            int flags = 0;
+            for (int i = 0; i < all.length; ++i) {
+                if ((flagset & (1 << i)) != 0) {
+                    flags |= all[i].flagValue;
+                }
+            }
+            for (int opt = 0; opt <= 2; opt++) {
+                for (String typ : new String[]{MIME_KIND_EVAL, MIME_KIND_COMPILE}) {
+                    mimeTypes.add(MIME_PREFIX + optFlagsToMime(opt, flags) + "-" + typ);
+                    mimeJavaStrings.add(String.format("\"%s\\%d\\u%04x-%s\"", MIME_PREFIX, opt, flags >> MIME_FLAG_SHIFTBY, typ));
+                }
+            }
+        }
+        HashSet<String> currentMimeTypes = new HashSet<>(List.of(PythonLanguage.class.getAnnotation(Registration.class).characterMimeTypes()));
+        if (!currentMimeTypes.containsAll(mimeTypes)) {
+            assert false : "Expected all of {" + String.join(", ", mimeJavaStrings) + "} in the PythonLanguage characterMimeTypes";
+        }
+    }
 
     public static final String MIME_TYPE_BYTECODE = "application/x-python-bytecode";
 
@@ -395,29 +427,26 @@ protected void initializeContext(PythonContext context) {
         context.initialize();
     }
 
-    public static String getCompileMimeType(int optimize, boolean futureAnnotations) {
-        String futureAnnField = futureAnnotations ? "1" : "0";
-        String compile = MIME_PREFIX + "compile-";
-        if (optimize <= 0) {
-            return compile + "0-" + futureAnnField;
-        } else if (optimize == 1) {
-            return compile + "1-" + futureAnnField;
-        } else {
-            return compile + "2-" + futureAnnField;
+    private static String optFlagsToMime(int optimize, int flags) {
+        if (optimize < 0) {
+            optimize = 0;
+        } else if (optimize > 2) {
+            optimize = 2;
         }
+        String optField = new String(new byte[]{(byte) optimize});
+        String flagField = new String(new int[]{(flags & FutureFeature.ALL_FLAGS) >> MIME_FLAG_SHIFTBY}, 0, 1);
+        assert flagField.length() == 1 : "flags in mime type ended up a surrogate";
+        return optField + flagField;
     }
 
-    // TODO inherit other implemented
-    public static String getEvalMimeType(int optimize, boolean futureAnnotations) {
-        String futureAnnField = futureAnnotations ? "1" : "0";
-        String eval = MIME_PREFIX + "eval-";
-        if (optimize <= 0) {
-            return eval + "0-" + futureAnnField;
-        } else if (optimize == 1) {
-            return eval + "1-" + futureAnnField;
-        } else {
-            return eval + "2-" + futureAnnField;
-        }
+    public static String getCompileMimeType(int optimize, int flags) {
+        String optFlags = optFlagsToMime(optimize, flags);
+        return MIME_PREFIX + optFlags + "-compile";
+    }
+
+    public static String getEvalMimeType(int optimize, int flags) {
+        String optFlags = optFlagsToMime(optimize, flags);
+        return MIME_PREFIX + optFlags + "-eval";
     }
 
     @Override
@@ -429,7 +458,7 @@ protected CallTarget parse(ParsingRequest request) {
                 throw new IllegalStateException("parse with arguments not allowed for interactive sources");
             }
             InputType inputType = source.isInteractive() ? InputType.SINGLE : InputType.FILE;
-            return parse(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames(), false);
+            return parse(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames(), EnumSet.noneOf(FutureFeature.class));
         }
         if (!request.getArgumentNames().isEmpty()) {
             throw new IllegalStateException("parse with arguments is only allowed for " + MIME_TYPE + " mime type");
@@ -468,34 +497,24 @@ protected CallTarget parse(ParsingRequest request) {
         String mime = source.getMimeType();
         String prefix = mime.substring(0, MIME_PREFIX.length());
         if (!prefix.equals(MIME_PREFIX)) {
-            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
-        }
-        String[] fields = mime.substring(MIME_PREFIX.length()).split("-");
-        if (fields.length != MIME_FIELD_COUNT) {
-            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
+            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + mime);
         }
-        String kind = fields[MIME_KIND_IDX];
+        String kind = mime.substring(MIME_KIND_START);
         InputType type;
         if (kind.equals(MIME_KIND_COMPILE)) {
             type = InputType.FILE;
         } else if (kind.equals(MIME_KIND_EVAL)) {
             type = InputType.EVAL;
         } else {
-            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
-        }
-        int optimize;
-        int futureAnnotations;
-        try {
-            optimize = Integer.parseInt(fields[MIME_OPTLEVEL_IDX]);
-            futureAnnotations = Integer.parseInt(fields[MIME_FUTURE_ANNOTATIONS_IDX]);
-        } catch (NumberFormatException e) {
-            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
+            throw CompilerDirectives.shouldNotReachHere("unknown compilation kind: " + kind + " from mime type: " + mime);
         }
-        if (0 > optimize || optimize > 2 || 0 > futureAnnotations || futureAnnotations > 2) {
-            throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
+        int optimize = mime.codePointAt(MIME_PREFIX.length());
+        int flags = mime.codePointAt(MIME_PREFIX.length() + 1) << MIME_FLAG_SHIFTBY;
+        if (0 > optimize || optimize > 2 || (flags & ~FutureFeature.ALL_FLAGS) != 0) {
+            throw CompilerDirectives.shouldNotReachHere("Invalid value for optlevel or flags: " + optimize + "," + flags + " from mime type: " + mime);
         }
         assert !source.isInteractive();
-        return parse(context, source, type, false, optimize, false, null, futureAnnotations == 1);
+        return parse(context, source, type, false, optimize, false, null, FutureFeature.fromFlags(flags));
     }
 
     private static Source tryLoadSource(PythonContext context, CodeUnit code, boolean internal, String path) {
@@ -507,13 +526,13 @@ private static Source tryLoadSource(PythonContext context, CodeUnit code, boolea
     }
 
     public RootCallTarget parse(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames,
-                    boolean futureAnnotations) {
+                    EnumSet<FutureFeature> futureFeatures) {
         RaisePythonExceptionErrorCallback errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
         try {
             Parser parser = Compiler.createParser(source.getCharacters().toString(), errorCb, type, interactiveTerminal);
             ModTy mod = (ModTy) parser.parse();
             assert mod != null;
-            return compileForBytecodeInterpreter(context, mod, source, topLevel, optimize, argumentNames, errorCb, futureAnnotations);
+            return compileForBytecodeInterpreter(context, mod, source, topLevel, optimize, argumentNames, errorCb, futureFeatures);
         } catch (PException e) {
             if (topLevel) {
                 PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e)).call();
@@ -524,7 +543,7 @@ public RootCallTarget parse(PythonContext context, Source source, InputType type
 
     @TruffleBoundary
     public RootCallTarget compileForBytecodeInterpreter(PythonContext context, ModTy mod, Source source, boolean topLevel, int optimize, List<String> argumentNames,
-                    RaisePythonExceptionErrorCallback errorCallback, boolean futureAnnotations) {
+                    RaisePythonExceptionErrorCallback errorCallback, EnumSet<FutureFeature> futureFeatures) {
         RaisePythonExceptionErrorCallback errorCb = errorCallback;
         if (errorCb == null) {
             errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
@@ -535,7 +554,7 @@ public RootCallTarget compileForBytecodeInterpreter(PythonContext context, ModTy
             if (hasArguments) {
                 mod = transformASTForExecutionWithArguments(argumentNames, mod);
             }
-            CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureAnnotations);
+            CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureFeatures);
             CodeUnit co = cu.assemble();
             RootNode rootNode = PBytecodeRootNode.create(this, co, source, errorCb);
             if (topLevel) {
@@ -620,7 +639,7 @@ public String getName() {
     @Override
     public ExecutableNode parse(InlineParsingRequest request) {
         PythonContext context = PythonContext.get(null);
-        RootCallTarget callTarget = parse(context, request.getSource(), InputType.EVAL, false, 0, false, null, false);
+        RootCallTarget callTarget = parse(context, request.getSource(), InputType.EVAL, false, 0, false, null, EnumSet.noneOf(FutureFeature.class));
         return new ExecutableNode(this) {
             @Child private GilNode gilNode = GilNode.create();
             @Child private GenericInvokeNode invokeNode = GenericInvokeNode.create();