implement named semaphores and _multiprocessing.sem_unlink

Author: timfel

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
 import java.util.function.Supplier;
 import java.util.logging.Level;
 
@@ -124,6 +125,14 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
 
     @CompilationFinal(dimensions = 1) private static final Object[] CONTEXT_INSENSITIVE_SINGLETONS = new Object[]{PNone.NONE, PNone.NO_VALUE, PEllipsis.INSTANCE, PNotImplemented.NOT_IMPLEMENTED};
 
+    /**
+     * Named semaphores are shared between all processes in a system, and they
+     * persist until the system is shut down, unless explicitly removed. We
+     * interpret this as meaning they all exist globally per language instance,
+     * that is, they are shared between different Contexts in the same engine.
+     */
+    public final ConcurrentHashMap<String, Semaphore> namedSemaphores = new ConcurrentHashMap<>();
+
     /*
      * We need to store this here, because the check is on the language and can come from a thread
      * that has no context, but we enable or disable threads with a context option. So we store this

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MultiprocessingModuleBuiltins.java

@@ -41,18 +41,25 @@
 package com.oracle.graal.python.builtins.modules;
 
 import java.util.List;
+import java.util.concurrent.Semaphore;
 
+import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.Builtin;
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.thread.PSemLock;
 import com.oracle.graal.python.builtins.objects.type.LazyPythonClass;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
+import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.nodes.util.CastToJavaIntNode;
+import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
 import com.oracle.graal.python.runtime.PythonCore;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
+import com.oracle.truffle.api.dsl.CachedLanguage;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
@@ -76,21 +83,68 @@ public void initialize(PythonCore core) {
     abstract static class ConstructSemLockNode extends PythonBuiltinNode {
         @Specialization
         PSemLock construct(LazyPythonClass cls, Object kindObj, Object valueObj, Object maxvalueObj, Object nameObj, Object unlinkObj,
+                        @Cached CastToJavaStringNode castNameNode,
                         @Cached CastToJavaIntNode castKindToIntNode,
                         @Cached CastToJavaIntNode castValueToIntNode,
                         @Cached CastToJavaIntNode castMaxvalueToIntNode,
-                        @Cached CastToJavaIntNode castUnlinkToIntNode) {
+                        @Cached CastToJavaIntNode castUnlinkToIntNode,
+                        @CachedLanguage PythonLanguage lang) {
             int kind = castKindToIntNode.execute(kindObj);
             if (kind != PSemLock.RECURSIVE_MUTEX && kind != PSemLock.SEMAPHORE) {
                 throw raise(PythonBuiltinClassType.ValueError, "unrecognized kind");
             }
             int value = castValueToIntNode.execute(valueObj);
-            int maxvalue = castMaxvalueToIntNode.execute(maxvalueObj);
+            castMaxvalueToIntNode.execute(maxvalueObj); // executed for the side-effect, but ignored on posix
+            Semaphore semaphore = newSemaphore(value);
             int unlink = castUnlinkToIntNode.execute(unlinkObj);
-            if (unlink != 0) {
-                throw raise(PythonBuiltinClassType.SystemError, "semaphore unlinking is not yet implemented");
+            String name = castNameNode.execute(nameObj);
+            if (unlink == 0) {
+                // CPython creates a named semaphore, and if unlink != 0 unlinks
+                // it directly so it cannot be access by other processes. We
+                // have to explicitly link it, so we do that here if we
+                // must. CPython always uses O_CREAT | O_EXCL for creating named
+                // semaphores, so a conflict raises.
+                if (semaphoreExists(lang, name)) {
+                    throw raise(PythonBuiltinClassType.FileExistsError, "Semaphore name taken: '%s'", name);
+                } else {
+                    semaphorePut(lang, semaphore, name);
+                }
             }
-            return factory().createSemLock(cls, kind, value, maxvalue, nameObj);
+            return factory().createSemLock(cls, kind, semaphore);
+        }
+
+        @TruffleBoundary
+        private static Object semaphorePut(PythonLanguage lang, Semaphore semaphore, String name) {
+            return lang.namedSemaphores.put(name, semaphore);
+        }
+
+        @TruffleBoundary
+        private static boolean semaphoreExists(PythonLanguage lang, String name) {
+            return lang.namedSemaphores.containsKey(name);
+        }
+
+        @TruffleBoundary
+        private static Semaphore newSemaphore(int value) {
+            return new Semaphore(value);
+        }
+    }
+
+    @GenerateNodeFactory
+    @Builtin(name = "sem_unlink", parameterNames = {"name"})
+    abstract static class SemUnlink extends PythonUnaryBuiltinNode {
+        @Specialization
+        PNone doit(String name,
+                        @CachedLanguage PythonLanguage lang) {
+            Semaphore prev = semaphoreRemove(name, lang);
+            if (prev == null) {
+                throw raise(PythonBuiltinClassType.FileNotFoundError, "No such file or directory: 'semaphores:/%s'", name);
+            }
+            return PNone.NONE;
+        }
+
+        @TruffleBoundary
+        private static Semaphore semaphoreRemove(String name, PythonLanguage lang) {
+            return lang.namedSemaphores.remove(name);
         }
     }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/thread/PSemLock.java

@@ -56,10 +56,9 @@ public final class PSemLock extends AbstractPythonLock {
     private int lastThreadID = -1;
     private int count;
 
-    @TruffleBoundary
-    public PSemLock(LazyPythonClass cls, int kind, int value) {
+    public PSemLock(LazyPythonClass cls, int kind, Semaphore sharedSemaphore) {
         super(cls);
-        semaphore = new Semaphore(value);
+        this.semaphore = sharedSemaphore;
         this.kind = kind;
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PythonObjectFactory.java

@@ -31,6 +31,7 @@
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.DirectoryStream;
 import java.util.Map;
+import java.util.concurrent.Semaphore;
 
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
@@ -823,8 +824,8 @@ public PThread createPythonThread(LazyPythonClass cls, Thread thread) {
         return trace(new PThread(cls, thread));
     }
 
-    public PSemLock createSemLock(LazyPythonClass cls, int kind, int value, @SuppressWarnings("unused") int maxvalue, @SuppressWarnings("unused") Object nameObj) {
-        return trace(new PSemLock(cls, kind, value));
+    public PSemLock createSemLock(LazyPythonClass cls, int kind, Semaphore sharedSemaphore) {
+        return trace(new PSemLock(cls, kind, sharedSemaphore));
     }
 
     public PScandirIterator createScandirIterator(LazyPythonClass cls, String path, DirectoryStream<TruffleFile> next) {