Free HPy native space if object dies

Author: fangerer

graalpython/com.oracle.graal.python.cext/hpy/hpy.c

@@ -527,3 +527,17 @@ void graal_hpy_write_ptr(void* object, HPy_ssize_t offset, void* value) {
 
 #undef WRAP
 #undef UNWRAP
+
+/* to be used from Java code only */
+int graal_hpy_bulk_free(void* ptrArray[], int64_t len) {
+	int64_t i;
+	void* obj;
+
+	for (i=0; i < len; i++) {
+		obj = ptrArray[i];
+		if (obj != NULL) {
+			free(obj);
+		}
+	}
+    return 0;
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyContext.java

@@ -48,6 +48,7 @@
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyContextFunctions.FunctionMode.OBJECT;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyDef.OBJECT_HPY_NATIVE_SPACE;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyDef.TYPE_HPY_BASICSIZE;
+import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyDef.TYPE_HPY_DESTROY;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNativeSymbols.GRAAL_HPY_DEF_GET_KIND;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNativeSymbols.GRAAL_HPY_DEF_GET_METH;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNativeSymbols.GRAAL_HPY_FROM_HPY_MODULE_DEF;
@@ -1387,6 +1388,7 @@ Object execute(Object[] arguments,
                         @Cached HPyRaiseNode raiseNode,
                         @Cached PythonObjectFactory factory,
                         @Cached WriteAttributeToObjectNode writeNativeSpaceNode,
+                        @Cached ReadAttributeFromObjectNode readAttributeFromObjectNode,
                         @Cached PCallHPyFunction callMallocNode,
                         @Cached PCallHPyFunction callWriteDataNode,
                         @Cached HPyAsHandleNode asHandleNode) throws ArityException {
@@ -1413,16 +1415,14 @@ Object execute(Object[] arguments,
                 long basicsize = (long) attrObj;
                 Object dataPtr = callMallocNode.call(context, GraalHPyNativeSymbols.GRAAL_HPY_CALLOC, basicsize, 1L);
                 writeNativeSpaceNode.execute(pythonObject, OBJECT_HPY_NATIVE_SPACE, dataPtr);
+                Object destroyFunc = readAttributeFromObjectNode.execute(type, TYPE_HPY_DESTROY);
+                context.createHandleReference(pythonObject, dataPtr, destroyFunc != PNone.NO_VALUE ? destroyFunc : null);
 
                 // write data pointer to out var
                 callWriteDataNode.call(context, GRAAL_HPY_WRITE_PTR, dataOutVar, 0L, dataPtr);
 
                 LOGGER.fine(() -> String.format("Allocated HPy object with native space of size %d at %s", basicsize, dataPtr));
                 // TODO(fa): add memory tracing
-
-                // TODO(fa): Here we allocated some native memory but we do never free it. We need
-                // to create a weakref to the Python object and if it dies, we need to free the
-                // native space.
             }
             return asHandleNode.execute(pythonObject);
         }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyContextFunctions.java

@@ -198,7 +198,6 @@ enum HPySlot {
         HPY_TP_REPR(66, SpecialMethodNames.__REPR__, HPyFuncSignature.REPRFUNC),
         HPY_NB_MATRIX_MULTIPLY(75, SpecialMethodNames.__MATMUL__, HPyFuncSignature.BINARYFUNC),
         HPY_NB_INPLACE_MATRIX_MULTIPLY(76, SpecialMethodNames.__IMATMUL__, HPyFuncSignature.BINARYFUNC),
-        // TODO(fa): use a hidden key ?
         HPY_TP_DESTROY(1000, TYPE_HPY_DESTROY, HPyFuncSignature.DESTROYFUNC);
 
         /** The corresponding C enum value. */

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyDef.java

@@ -40,6 +40,9 @@
  */
 package com.oracle.graal.python.builtins.objects.cext.hpy;
 
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
+
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.objects.PythonAbstractObject;
 import com.oracle.graal.python.builtins.objects.cext.PythonNativeWrapperLibrary;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyHandle.java

@@ -117,4 +117,5 @@ public abstract class GraalHPyNativeSymbols {
     public static final String GRAAL_HPY_WRITE_HPY_SSIZE_T = "graal_hpy_write_HPy_ssize_t";
     public static final String GRAAL_HPY_WRITE_PTR = "graal_hpy_write_ptr";
 
+    public static final String GRAAL_HPY_BULK_FREE = "graal_hpy_bulk_free";
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyNativeSymbols.java

@@ -41,6 +41,7 @@
 package com.oracle.graal.python.builtins.objects.cext.hpy;
 
 import static com.oracle.graal.python.builtins.PythonBuiltinClassType.SystemError;
+import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyDef.HPySlot.HPY_TP_DESTROY;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyDef.HPySlot.HPY_TP_NEW;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNativeSymbols.GRAAL_HPY_DEF_GET_GETSET;
 import static com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyNativeSymbols.GRAAL_HPY_DEF_GET_KIND;
@@ -808,8 +809,15 @@ static HPyProperty doIt(GraalHPyContext context, Object enclosingType, Object sl
 
             String methodNameStr = methodName instanceof HiddenKey ? ((HiddenKey) methodName).getName() : (String) methodName;
 
-            PBuiltinFunction function = HPyExternalFunctionNodes.createWrapperFunction(language, slot.getSignature(), methodNameStr, methodFunctionPointer,
-                            HPY_TP_NEW.equals(slot) ? null : enclosingType, factory);
+            Object function;
+            if (HPY_TP_DESTROY.equals(slot)) {
+                // special case: DESTROYFUNC
+                // This won't be usable from Python, so we just store the bare pointer object into
+                // the hidden attribute.
+                function = methodFunctionPointer;
+            } else {
+                function = HPyExternalFunctionNodes.createWrapperFunction(language, slot.getSignature(), methodNameStr, methodFunctionPointer, HPY_TP_NEW.equals(slot) ? null : enclosingType, factory);
+            }
             return new HPyProperty(methodName, function);
         }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/GraalHPyNodes.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.objects.cext.hpy;
+
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.cext.hpy.GraalHPyContext.GraalHPyHandleReference;
+import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.TruffleObject;
+import com.oracle.truffle.api.library.ExportLibrary;
+import com.oracle.truffle.api.library.ExportMessage;
+
+@ExportLibrary(InteropLibrary.class)
+final class NativeSpaceArrayWrapper implements TruffleObject {
+
+    final GraalHPyHandleReference[] data;
+
+    public NativeSpaceArrayWrapper(GraalHPyHandleReference[] data) {
+        this.data = data;
+    }
+
+    @ExportMessage
+    boolean hasArrayElements() {
+        return true;
+    }
+
+    @ExportMessage
+    long getArraySize() {
+        return data.length;
+    }
+
+    @ExportMessage
+    boolean isArrayElementReadable(long i) {
+        return i < data.length;
+    }
+
+    @ExportMessage
+    Object readArrayElement(long i) {
+        GraalHPyHandleReference ref = data[(int) i];
+        if (ref != null) {
+            return ref.getNativeSpace();
+        }
+        // return something that responds to 'isNull' with 'true'
+        return PNone.NO_VALUE;
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/hpy/NativeSpaceArrayWrapper.java

@@ -126,7 +126,7 @@ public final void execute(PythonContext context) {
         }
     }
 
-    private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(3, new ThreadFactory() {
+    private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4, new ThreadFactory() {
         public Thread newThread(Runnable r) {
             Thread t = Executors.defaultThreadFactory().newThread(r);
             t.setDaemon(true);