Keep native libraries alive as long as the CApiContext

Author: fangerer

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java

@@ -49,6 +49,8 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
@@ -98,7 +100,6 @@
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
-import com.oracle.truffle.api.RootCallTarget;
 import com.oracle.truffle.api.TruffleFile;
 import com.oracle.truffle.api.TruffleLanguage.Env;
 import com.oracle.truffle.api.TruffleLogger;
@@ -186,7 +187,21 @@ private record ClosureInfo(Object closure, Object delegate, Object executable, l
     private final HashMap<Object, ClosureInfo> callableClosureByExecutable = new HashMap<>();
     private final HashMap<Long, ClosureInfo> callableClosures = new HashMap<>();
     private Object nativeLibrary;
-    public RootCallTarget signatureContainer;
+
+    /**
+     * This list holds a strong reference to all loaded extension libraries to keep the library
+     * objects alive. This is necessary because NFI will {@code dlclose} the library (and thus
+     * {@code munmap} all code) if the library object is no longer reachable. However, it can happen
+     * that we still store raw function pointers (as Java {@code long} values) in a native object
+     * that is referenced by a
+     * {@link com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.NativeObjectReference}.
+     * For example, the {code tp_dealloc} functions may be executed a long time after all managed
+     * objects of the extension died and the native library has been {@code dlclosed}'d.
+     *
+     * Since we have no control over the timing when certain garbage will be collected, we need to
+     * ensure that the code is still mapped.
+     */
+    private final List<Object> loadedExtensions = new LinkedList<>();
 
     public static TruffleLogger getLogger(Class<?> clazz) {
         return PythonLanguage.getLogger(LOGGER_CAPI_NAME + "." + clazz.getSimpleName());
@@ -209,6 +224,11 @@ public long getAndIncMaxModuleNumber() {
         return maxModuleNumber++;
     }
 
+    @TruffleBoundary
+    void addLoadedExtensionLibrary(Object nativeLibrary) {
+        loadedExtensions.add(nativeLibrary);
+    }
+
     @TruffleBoundary
     public static Object asHex(Object ptr) {
         if (ptr instanceof Number) {
@@ -631,6 +651,7 @@ public Object initCApiModule(Node location, Object sharedLibrary, TruffleString
             PythonModule module = (PythonModule) result;
             module.setAttribute(T___FILE__, spec.path);
             module.setAttribute(T___LIBRARY__, sharedLibrary);
+            addLoadedExtensionLibrary(sharedLibrary);
 
             // add to 'sys.modules'
             PDict sysModules = context.getSysModules();

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java

@@ -1921,6 +1921,7 @@ static Object doGeneric(CApiContext capiContext, ModuleSpec moduleSpec, Object m
 
             writeAttrNode.execute(module, SpecialAttributeNames.T___DOC__, mDoc);
             writeAttrNode.execute(module, SpecialAttributeNames.T___LIBRARY__, library);
+            capiContext.addLoadedExtensionLibrary(library);
             return module;
         }
     }