Share named semaphores between both contexts sharing an engine and also main and children contexts in the multiprocessing scenario

Author: steve-s

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
 import java.util.logging.Level;
 
 import org.graalvm.options.OptionDescriptors;
@@ -193,6 +194,20 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
 
     @CompilationFinal(dimensions = 1) private static final Object[] CONTEXT_INSENSITIVE_SINGLETONS = new Object[]{PNone.NONE, PNone.NO_VALUE, PEllipsis.INSTANCE, PNotImplemented.NOT_IMPLEMENTED};
 
+    /**
+     * Named semaphores are shared between all processes in a system, and they persist until the
+     * system is shut down, unless explicitly removed. We interpret this as meaning they all exist
+     * globally per language instance, that is, they are shared between different Contexts in the
+     * same engine.
+     *
+     * Top level contexts use this map to initialize their shared multiprocessing data. Inner
+     * children contexts created for the multiprocessing module ignore this map in
+     * {@link PythonLanguage} and instead inherit it in the shared multiprocessing data from their
+     * parent context. This way, the child inner contexts do not have to run in the same engine
+     * (have the same language instance), but can still share the named semaphores.
+     */
+    public final ConcurrentHashMap<String, Semaphore> namedSemaphores = new ConcurrentHashMap<>();
+
     @CompilationFinal(dimensions = 1) private volatile Object[] engineOptionsStorage;
     @CompilationFinal private volatile OptionValues engineOptions;
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -554,6 +554,10 @@ public static final class SharedMultiprocessingData {
          */
         private final Map<Integer, Integer> fdRefCount = new HashMap<>();
 
+        public SharedMultiprocessingData(ConcurrentHashMap<String, Semaphore> namedSemaphores) {
+            this.namedSemaphores = namedSemaphores;
+        }
+
         /**
          * Increases reference count for the given file descriptor.
          */
@@ -676,11 +680,9 @@ private boolean isClosed(int fd) {
         }
 
         /**
-         * Named semaphores are shared between all processes in a system, and they persist until the
-         * system is shut down, unless explicitly removed. We interpret this as meaning they all
-         * exist globally per the main context and all its children.
+         * @see PythonLanguage#namedSemaphores
          */
-        private final ConcurrentHashMap<String, Semaphore> namedSemaphores = new ConcurrentHashMap<>();
+        private final ConcurrentHashMap<String, Semaphore> namedSemaphores;
 
         @TruffleBoundary
         public void putNamedSemaphore(String name, Semaphore sem) {
@@ -732,7 +734,7 @@ public PythonContext(PythonLanguage language, TruffleLanguage.Env env, Python3Co
         this.core = core;
         this.env = env;
         this.childContextData = (ChildContextData) env.getConfig().get(CHILD_CONTEXT_DATA);
-        this.sharedMultiprocessingData = this.childContextData == null ? new SharedMultiprocessingData() : childContextData.parentCtx.sharedMultiprocessingData;
+        this.sharedMultiprocessingData = this.childContextData == null ? new SharedMultiprocessingData(language.namedSemaphores) : childContextData.parentCtx.sharedMultiprocessingData;
         this.handler = new AsyncHandler(this);
         this.sharedFinalizer = new AsyncHandler.SharedFinalizer(this);
         this.optionValues = PythonOptions.createOptionValuesStorage(env);