Multiprocessing spawn improvements

* Use new inner context builder
* Reuse parsed and compiled code for worker context entrypoint

Author: steve-s

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -1035,11 +1035,15 @@ public SharedMultiprocessingData getSharedMultiprocessingData() {
 
     public long spawnTruffleContext(int fd, int sentinel, int[] fdsToKeep) {
         ChildContextData data = new ChildContextData(isChildContext() ? childContextData.parentCtx : this);
-
-        Builder builder = data.parentCtx.env.newContextBuilder().config(PythonContext.CHILD_CONTEXT_DATA, data);
+        Builder builder = data.parentCtx.env.newInnerContextBuilder().//
+                        forceSharing(getOption(PythonOptions.ForceSharingForInnerContexts)).//
+                        inheritAllAccess(true).//
+                        initializeCreatorContext(true).//
+                        // TODO always force java posix in spawned: test_multiprocessing_spawn fails
+                        // with that. Gives "OSError: [Errno 9] Bad file number"
+                        // option("python.PosixModuleBackend", "java").//
+                        config(PythonContext.CHILD_CONTEXT_DATA, data);
         Thread thread = data.parentCtx.env.createThread(new ChildContextThread(fd, sentinel, data, builder));
-
-        // TODO always force java posix in spawned
         long tid = thread.getId();
         getSharedMultiprocessingData().putChildContextThread(tid, thread);
         getSharedMultiprocessingData().putChildContextData(tid, data);
@@ -1065,6 +1069,11 @@ public synchronized List<Integer> getChildContextFDs() {
     }
 
     private static class ChildContextThread implements Runnable {
+        private static final TruffleLogger MULTIPROCESSING_LOGGER = PythonLanguage.getLogger(ChildContextThread.class);
+        private static final Source MULTIPROCESSING_SOURCE = Source.newBuilder(PythonLanguage.ID,
+                        "from multiprocessing.spawn import spawn_truffleprocess; spawn_truffleprocess(fd, sentinel)",
+                        "<spawned-child-context>").internal(true).build();
+
         private final int fd;
         private final ChildContextData data;
         private final Builder builder;
@@ -1080,28 +1089,24 @@ public ChildContextThread(int fd, int sentinel, ChildContextData data, Builder b
         @Override
         public void run() {
             try {
-                LOGGER.fine("starting spawned child context");
-                Source source = Source.newBuilder(PythonLanguage.ID,
-                                "from multiprocessing.spawn import spawn_truffleprocess; spawn_truffleprocess(" + fd + ", " + sentinel + ")",
-                                "<spawned-child-context>").internal(true).build();
-                CallTarget ct;
+                MULTIPROCESSING_LOGGER.fine("starting spawned child context");
                 TruffleContext ctx = builder.build();
                 data.setTruffleContext(ctx);
                 Object parent = ctx.enter(null);
-                ct = PythonContext.get(null).getEnv().parsePublic(source);
+                CallTarget ct = PythonContext.get(null).getEnv().parsePublic(MULTIPROCESSING_SOURCE, "fd", "sentinel");
                 try {
                     data.running.countDown();
-                    Object res = ct.call();
+                    Object res = ct.call(fd, sentinel);
                     int exitCode = CastToJavaIntLossyNode.getUncached().execute(res);
                     data.setExitCode(exitCode);
                 } finally {
                     ctx.leave(null, parent);
                     if (data.compareAndSetExiting(false, true)) {
                         try {
                             ctx.close();
-                            LOGGER.log(Level.FINE, "closed spawned child context");
+                            MULTIPROCESSING_LOGGER.log(Level.FINE, "closed spawned child context");
                         } catch (Throwable t) {
-                            LOGGER.log(Level.FINE, t, () -> "exception while closing spawned child context");
+                            MULTIPROCESSING_LOGGER.log(Level.FINE, "exception while closing spawned child context", t);
                         }
                     }
                     data.parentCtx.sharedMultiprocessingData.closePipe(sentinel);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -286,6 +286,9 @@ private PythonOptions() {
     @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = "Propagate append operations to lists created as literals back to where they were created, to inform overallocation to avoid having to grow them later.") //
     public static final OptionKey<Boolean> OverallocateLiteralLists = new OptionKey<>(true);
 
+    @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = "Forces AST sharing for inner contexts.") //
+    public static final OptionKey<Boolean> ForceSharingForInnerContexts = new OptionKey<>(true);
+
     @EngineOption @Option(category = OptionCategory.USER, usageSyntax = "true|false", help = "Emulate some Jython features that can cause performance degradation") //
     public static final OptionKey<Boolean> EmulateJython = new OptionKey<>(false);
 

graalpython/lib-python/3/multiprocessing/spawn.py

@@ -144,8 +144,7 @@ def spawn_truffleprocess(fd, parent_sentinel):
         self = reduction.pickle.load(bytesIO)
     finally:
         del process.current_process()._inheriting
-    exitcode = self._bootstrap(parent_sentinel)    
-    sys.exit(exitcode)
+    return self._bootstrap(parent_sentinel)
 # End Truffle change
 
 def _check_not_importing_main():