finalize importlib freezing

- fix generating FrozenModules.java for the special case of an stdlib module
  that has only alias definitions
- remove _imp, _frozen_importlib, and final_patches

Author: timfel

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -69,7 +69,7 @@
             "ntpath",
             "posixpath",
             # We must explicitly mark os.path as a frozen module
-            f'{"ntpath" if os.name == "nt" else "posixpath"} : os.path",
+            ("ntpath" if os.name == "nt" else "posixpath") + " : os.path",
             "os",
             "site",
             "stat",
@@ -526,7 +526,10 @@ def freeze_module(src):
 def write_frozen_modules_map(out_file, modules):
     out_file.write("    private static final class Map {\n")
     for module in modules:
-        if not module.isalias or not module.orig:
+        if (not module.isalias or
+            not module.orig or
+            not any(module.orig == m.orig for m in modules if m != module)
+        ):
             ispkg = "true" if module.ispkg else "false"
             out_file.write(
                 f'        private static final PythonFrozenModule {module.symbol} = new PythonFrozenModule("{module.symbol}", "{module.frozenid}", {ispkg});\n'

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -26,12 +26,13 @@
 package com.oracle.graal.python.builtins;
 
 import static com.oracle.graal.python.builtins.PythonBuiltinClassType.IndentationError;
+import static com.oracle.graal.python.builtins.PythonBuiltinClassType.SyntaxError;
 import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TabError;
 import static com.oracle.graal.python.builtins.objects.exception.SyntaxErrorBuiltins.SYNTAX_ERROR_ATTR_FACTORY;
 import static com.oracle.graal.python.nodes.BuiltinNames.MODULES;
 import static com.oracle.graal.python.nodes.BuiltinNames.PRINT;
 import static com.oracle.graal.python.nodes.SpecialAttributeNames.__PACKAGE__;
-import static com.oracle.graal.python.runtime.exception.PythonErrorType.SyntaxError;
+import static com.oracle.graal.python.nodes.SpecialMethodNames.__REPR__;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -357,9 +358,7 @@ private static String[] initializeCoreFiles() {
         // Order matters!
         List<String> coreFiles = new ArrayList<>(Arrays.asList(
                         "type",
-                        "_imp",
                         "function",
-                        "_frozen_importlib",
                         "__graalpython__",
                         "_weakref",
                         "faulthandler",
@@ -386,8 +385,6 @@ private static String[] initializeCoreFiles() {
                 }
             }
         }
-        // must be last
-        coreFiles.add("final_patches");
         return coreFiles.toArray(new String[coreFiles.size()]);
     }
 
@@ -829,7 +826,7 @@ private void initializeJavaCore() {
     }
 
     private void initializeImportlib() {
-        PythonModule bootstrap = (PythonModule) ImpModuleBuiltins.importFrozenModuleObject(this, "_frozen_importlib");
+        PythonModule bootstrap = (PythonModule) ImpModuleBuiltins.importFrozenModuleObject(this, "_frozen_importlib", false);
 
         PyObjectCallMethodObjArgs callNode = PyObjectCallMethodObjArgs.getUncached();
         WriteAttributeToDynamicObjectNode writeNode = WriteAttributeToDynamicObjectNode.getUncached();
@@ -843,8 +840,8 @@ private void initializeImportlib() {
             bootstrap = createModule("importlib._bootstrap");
             writeNode.execute(bootstrap, __PACKAGE__, "importlib");
             addBuiltinModule("_frozen_importlib", bootstrap);
-            loadFile("importlib/_bootstrap_external.py", getContext().getStdlibHome());
-            loadFile("importlib/_bootstrap.py", getContext().getStdlibHome());
+            loadFile("importlib/_bootstrap_external", getContext().getStdlibHome(), bootstrapExternal);
+            loadFile("importlib/_bootstrap", getContext().getStdlibHome(), bootstrap);
         }
 
         callNode.execute(null, bootstrap, "_install", getSysModule(), lookupBuiltinModule("_imp"));
@@ -853,8 +850,11 @@ private void initializeImportlib() {
         importFunc = (PFunction) readNode.execute(bootstrap, "__import__");
         importlib = bootstrap;
 
+        writeNode.execute(lookupType(PythonBuiltinClassType.PythonModule), __REPR__, readNode.execute(bootstrap, "_module_repr"));
+
         // __package__ needs to be set and doesn't get set by _bootstrap setup
         writeNode.execute(bootstrap, __PACKAGE__, "importlib");
+
         PythonModule bootstrapExternal = (PythonModule) getSysModules().getItem("_frozen_importlib_external");
         writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
     }
@@ -1101,16 +1101,20 @@ private Source getInternalSource(String basename, String prefix) {
     }
 
     private void loadFile(String s, String prefix) {
-        Supplier<CallTarget> getCode = () -> {
-            Source source = getInternalSource(s, prefix);
-            return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));
-        };
-        RootCallTarget callTarget = (RootCallTarget) getLanguage().cacheCode(s, getCode);
         PythonModule mod = lookupBuiltinModule(s);
         if (mod == null) {
             // use an anonymous module for the side-effects
             mod = factory().createPythonModule("__anonymous__");
         }
+        loadFile(s, prefix, mod);
+    }
+
+    private void loadFile(String s, String prefix, PythonModule mod) {
+        Supplier<CallTarget> getCode = () -> {
+            Source source = getInternalSource(s, prefix);
+            return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));
+        };
+        RootCallTarget callTarget = (RootCallTarget) getLanguage().cacheCode(s, getCode);
         GenericInvokeNode.getUncached().execute(callTarget, PArguments.withGlobals(mod));
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java

@@ -465,8 +465,7 @@ protected ArgumentClinicProvider getArgumentClinic() {
 
         @Specialization
         boolean run(String name) {
-            PythonFrozenModule mod = FrozenModules.lookup(name);
-            return mod != null && mod.getCode() != null;
+            return findFrozen(name).status == FROZEN_OKAY;
         }
     }
 
@@ -618,7 +617,7 @@ protected ArgumentClinicProvider getArgumentClinic() {
 
         @Specialization
         Object run(String name) {
-            return importFrozenModuleObject(getCore(), name);
+            return importFrozenModuleObject(getCore(), name, true);
         }
     }
 
@@ -627,7 +626,7 @@ Object run(String name) {
      * imported module, null, or raises a Python exception.
      */
     @TruffleBoundary
-    public static Object importFrozenModuleObject(Python3Core core, String name) {
+    public static Object importFrozenModuleObject(Python3Core core, String name, boolean doRaise) {
         FrozenResult result = findFrozen(name);
         FrozenStatus status = result.status;
         FrozenInfo info = result.info;
@@ -638,7 +637,11 @@ public static Object importFrozenModuleObject(Python3Core core, String name) {
             case FROZEN_BAD_NAME:
                 return null;
             default:
-                raiseFrozenError(status, name, PRaiseNode.getUncached());
+                if (doRaise) {
+                    raiseFrozenError(status, name, PRaiseNode.getUncached());
+                } else {
+                    return null;
+                }
         }
 
         PCode code = (PCode) MarshalModuleBuiltins.Marshal.load(info.data, info.size);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java

@@ -42,6 +42,8 @@
 public final class FrozenModules {
 
     private static final class Map {
+        private static final PythonFrozenModule IMPORTLIB__BOOTSTRAP = new PythonFrozenModule("IMPORTLIB__BOOTSTRAP", "importlib._bootstrap", false);
+        private static final PythonFrozenModule IMPORTLIB__BOOTSTRAP_EXTERNAL = new PythonFrozenModule("IMPORTLIB__BOOTSTRAP_EXTERNAL", "importlib._bootstrap_external", false);
         private static final PythonFrozenModule ABC = new PythonFrozenModule("ABC", "abc", false);
         private static final PythonFrozenModule CODECS = new PythonFrozenModule("CODECS", "codecs", false);
         private static final PythonFrozenModule ENCODINGS = new PythonFrozenModule("ENCODINGS", "encodings", true);
@@ -200,6 +202,8 @@ private static final class Map {
 
     public static final PythonFrozenModule lookup(String name) {
         switch (name) {
+            case "_frozen_importlib": return Map.IMPORTLIB__BOOTSTRAP;
+            case "_frozen_importlib_external": return Map.IMPORTLIB__BOOTSTRAP_EXTERNAL;
             case "abc": return Map.ABC;
             case "codecs": return Map.CODECS;
             case "encodings": return Map.ENCODINGS;
@@ -351,6 +355,7 @@ public static final PythonFrozenModule lookup(String name) {
             case "genericpath": return Map.GENERICPATH;
             case "ntpath": return Map.NTPATH;
             case "posixpath": return Map.POSIXPATH;
+            case "os.path": return Map.POSIXPATH;
             case "os": return Map.OS;
             case "site": return Map.SITE;
             case "stat": return Map.STAT;