Skip to content

Commit 2af3a19

Browse files
msimacekmsimacek
committed
Add cleanup for NativeSequenceStorage
1 parent b6b48d5 commit 2af3a19

File tree

7 files changed

+102
-22
lines changed

7 files changed

+102
-22
lines changed

‎graalpython/com.oracle.graal.python.cext/src/capi.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,17 @@ PRIMITIVE_ARRAY_TO_NATIVE(Long, int64_t, i64, polyglot_as_i64);
894894
PRIMITIVE_ARRAY_TO_NATIVE(Double, double, double, polyglot_as_double);
895895
PRIMITIVE_ARRAY_TO_NATIVE(Object, PyObjectPtr, PyObjectPtr, (PyObjectPtr));
896896

897+
void PyTruffle_PrimitiveArrayFree(void* array) {
898+
free(array);
899+
}
900+
901+
void PyTruffle_ObjectArrayFree(PyObject** array, int size) {
902+
for (int i = 0; i < size; i++) {
903+
Py_DECREF(array[i]);
904+
}
905+
free(array);
906+
}
907+
897908
PyAPI_FUNC(Py_ssize_t) PyTruffle_Object_Size(PyObject *op) {
898909
return ((PyVarObject*)op)->ob_size;
899910
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextBuiltins.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1203,7 +1203,7 @@ Object wrap(Object bufferStructPointer, Object ownerObj, Object lenObj,
12031203
}
12041204
}
12051205
}
1206-
Object buffer = new NativeSequenceStorage(bufPointer, len, len, SequenceStorage.ListStorageType.Byte);
1206+
Object buffer = NativeSequenceStorage.create(bufPointer, len, len, SequenceStorage.ListStorageType.Byte, false);
12071207
int flags = initFlagsNode.execute(ndim, itemsize, shape, strides, suboffsets);
12081208
BufferLifecycleManager bufferLifecycleManager = null;
12091209
if (!lib.isNull(bufferStructPointer)) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ public enum NativeCAPISymbol implements NativeCExtSymbol {
154154
FUN_PY_TRUFFLE_LONG_ARRAY_REALLOC("PyTruffle_LongArrayRealloc"),
155155
FUN_PY_TRUFFLE_DOUBLE_ARRAY_REALLOC("PyTruffle_DoubleArrayRealloc"),
156156
FUN_PY_TRUFFLE_OBJECT_ARRAY_REALLOC("PyTruffle_ObjectArrayRealloc"),
157+
FUN_PY_TRUFFLE_PRIMITIVE_ARRAY_FREE("PyTruffle_PrimitiveArrayFree"),
158+
FUN_PY_TRUFFLE_OBJECT_ARRAY_FREE("PyTruffle_ObjectArrayFree"),
157159
FUN_PY_OBJECT_GENERIC_GET_DICT("_PyObject_GenericGetDict"),
158160
FUN_PY_OBJECT_NEW("PyTruffle_Object_New"),
159161
FUN_GET_THREAD_STATE_TYPE_ID("get_thread_state_typeid"),

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java

Lines changed: 54 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,28 +44,32 @@
4444
import java.lang.ref.WeakReference;
4545
import java.util.ArrayList;
4646
import java.util.HashMap;
47+
import java.util.HashSet;
48+
import java.util.Set;
4749
import java.util.WeakHashMap;
4850
import java.util.logging.Level;
4951

5052
import com.oracle.graal.python.builtins.objects.PNone;
5153
import com.oracle.graal.python.builtins.objects.PythonAbstractObject;
5254
import com.oracle.graal.python.builtins.objects.cext.PythonAbstractNativeObject;
55+
import com.oracle.graal.python.builtins.objects.cext.capi.CApiContext;
5356
import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.FromCharPointerNode;
5457
import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.PCallCapiFunction;
5558
import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodesFactory.FromCharPointerNodeGen;
5659
import com.oracle.graal.python.builtins.objects.cext.capi.DynamicObjectNativeWrapper.PrimitiveNativeWrapper;
57-
import com.oracle.graal.python.builtins.objects.cext.capi.CApiContext;
5860
import com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbol;
5961
import com.oracle.graal.python.builtins.objects.cext.capi.PythonNativePointer;
6062
import com.oracle.graal.python.builtins.objects.cext.capi.PythonNativeWrapper;
6163
import com.oracle.graal.python.builtins.objects.cext.capi.TruffleObjectNativeWrapper;
6264
import com.oracle.graal.python.builtins.objects.getsetdescriptor.DescriptorDeleteMarker;
6365
import com.oracle.graal.python.runtime.PythonContext;
66+
import com.oracle.graal.python.runtime.sequence.storage.NativeSequenceStorage;
67+
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage.ListStorageType;
6468
import com.oracle.graal.python.util.PythonUtils;
6569
import com.oracle.truffle.api.CompilerAsserts;
6670
import com.oracle.truffle.api.CompilerDirectives;
67-
import com.oracle.truffle.api.TruffleLogger;
6871
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
72+
import com.oracle.truffle.api.TruffleLogger;
6973
import com.oracle.truffle.api.dsl.Cached;
7074
import com.oracle.truffle.api.interop.InteropLibrary;
7175
import com.oracle.truffle.api.interop.TruffleObject;
@@ -94,6 +98,7 @@ public static final class HandleContext {
9498
public final HashMap<Long, IdReference<?>> nativeLookup = new HashMap<>();
9599
public final WeakHashMap<Object, WeakReference<PythonAbstractNativeObject>> managedNativeLookup = new WeakHashMap<>();
96100
public final ArrayList<PythonObjectReference> nativeHandles = new ArrayList<>();
101+
public final Set<NativeStorageReference> nativeStorageReferences = new HashSet<>();
97102

98103
public final ReferenceQueue<Object> referenceQueue = new ReferenceQueue<>();
99104

@@ -146,7 +151,7 @@ public NativeObjectReference(PythonAbstractNativeObject referent, long pointer)
146151
this.pointer = pointer;
147152
referent.ref = this;
148153
assert (pointer & 7) == 0;
149-
LOGGER.finer(() -> PythonUtils.formatJString("new NativeObjectReference reference to %s", referent));
154+
LOGGER.finer(() -> PythonUtils.formatJString("new NativeObjectReference<%s> to %s", Long.toHexString(pointer), referent));
150155
}
151156

152157
@Override
@@ -155,6 +160,41 @@ public String toString() {
155160
}
156161
}
157162

163+
public static final class NativeStorageReference extends IdReference<NativeSequenceStorage> {
164+
private final ListStorageType type;
165+
private Object ptr;
166+
private int size;
167+
168+
public NativeStorageReference(NativeSequenceStorage storage) {
169+
super(storage);
170+
type = storage.getElementType();
171+
ptr = storage.getPtr();
172+
LOGGER.finer(() -> PythonUtils.formatJString("new NativeStorageReference<%s>", ptr));
173+
}
174+
175+
public Object getPtr() {
176+
return ptr;
177+
}
178+
179+
public void setPtr(Object ptr) {
180+
this.ptr = ptr;
181+
}
182+
183+
public int getSize() {
184+
return size;
185+
}
186+
187+
public void setSize(int size) {
188+
this.size = size;
189+
}
190+
}
191+
192+
public static void registerNativeSequenceStorage(NativeSequenceStorage storage) {
193+
NativeStorageReference ref = new NativeStorageReference(storage);
194+
storage.setReference(ref);
195+
getContext().nativeStorageReferences.add(ref);
196+
}
197+
158198
@TruffleBoundary
159199
public static void pollReferenceQueue() {
160200
HandleContext context = getContext();
@@ -178,8 +218,7 @@ public static void pollReferenceQueue() {
178218
assert context.referenceQueuePollActive;
179219
}
180220
count++;
181-
if (entry instanceof PythonObjectReference) {
182-
PythonObjectReference reference = (PythonObjectReference) entry;
221+
if (entry instanceof PythonObjectReference reference) {
183222
LOGGER.finer(() -> PythonUtils.formatJString("releasing PythonObjectReference %s", reference));
184223

185224
if (HandleTester.pointsToPyHandleSpace(reference.pointer)) {
@@ -190,8 +229,7 @@ public static void pollReferenceQueue() {
190229
assert nativeLookupGet(context, reference.pointer) != null : Long.toHexString(reference.pointer);
191230
nativeLookupRemove(context, reference.pointer);
192231
}
193-
} else {
194-
NativeObjectReference reference = (NativeObjectReference) entry;
232+
} else if (entry instanceof NativeObjectReference reference) {
195233
LOGGER.finer(() -> PythonUtils.formatJString("releasing NativeObjectReference %s", reference));
196234
nativeLookupRemove(context, reference.pointer);
197235
Object object = reference.object;
@@ -203,6 +241,15 @@ public static void pollReferenceQueue() {
203241
}
204242
}
205243
PCallCapiFunction.getUncached().call(NativeCAPISymbol.FUN_SUBREF, object, PythonNativeWrapper.MANAGED_REFCNT);
244+
} else if (entry instanceof NativeStorageReference reference) {
245+
LOGGER.finer(() -> PythonUtils.formatJString("releasing NativeStorageReference %s", reference));
246+
context.nativeStorageReferences.remove(entry);
247+
if (reference.type == ListStorageType.Generic) {
248+
PCallCapiFunction.getUncached().call(NativeCAPISymbol.FUN_PY_TRUFFLE_OBJECT_ARRAY_FREE, reference.ptr, reference.size);
249+
} else {
250+
assert reference.type != ListStorageType.Uninitialized;
251+
PCallCapiFunction.getUncached().call(NativeCAPISymbol.FUN_PY_TRUFFLE_PRIMITIVE_ARRAY_FREE, reference.ptr);
252+
}
206253
}
207254
}
208255
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/SequenceStorageNodes.java

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,25 +1596,26 @@ public abstract static class StorageToNativeNode extends Node {
15961596
@Specialization
15971597
static NativeSequenceStorage doByte(byte[] arr, int length,
15981598
@Exclusive @Cached PCallCapiFunction callNode) {
1599-
return new NativeSequenceStorage(callNode.call(FUN_PY_TRUFFLE_BYTE_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Byte);
1599+
return NativeSequenceStorage.create(callNode.call(FUN_PY_TRUFFLE_BYTE_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Byte, true);
16001600
}
16011601

16021602
@Specialization
16031603
static NativeSequenceStorage doInt(int[] arr, int length,
16041604
@Exclusive @Cached PCallCapiFunction callNode) {
1605-
return new NativeSequenceStorage(callNode.call(FUN_PY_TRUFFLE_INT_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Int);
1605+
return NativeSequenceStorage.create(callNode.call(FUN_PY_TRUFFLE_INT_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Int, true);
16061606
}
16071607

16081608
@Specialization
16091609
static NativeSequenceStorage doLong(long[] arr, int length,
16101610
@Exclusive @Cached PCallCapiFunction callNode) {
1611-
return new NativeSequenceStorage(callNode.call(FUN_PY_TRUFFLE_LONG_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Long);
1611+
return NativeSequenceStorage.create(callNode.call(FUN_PY_TRUFFLE_LONG_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Long, true);
16121612
}
16131613

16141614
@Specialization
16151615
static NativeSequenceStorage doDouble(double[] arr, int length,
16161616
@Exclusive @Cached PCallCapiFunction callNode) {
1617-
return new NativeSequenceStorage(callNode.call(FUN_PY_TRUFFLE_DOUBLE_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Double);
1617+
return NativeSequenceStorage.create(callNode.call(FUN_PY_TRUFFLE_DOUBLE_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), arr), arr.length), length, arr.length, ListStorageType.Double,
1618+
true);
16181619
}
16191620

16201621
@Specialization
@@ -1625,9 +1626,8 @@ static NativeSequenceStorage doObject(Object[] arr, int length,
16251626
for (int i = 0; i < wrappedValues.length; i++) {
16261627
wrappedValues[i] = toSulongNode.execute(arr[i]);
16271628
}
1628-
return new NativeSequenceStorage(callNode.call(FUN_PY_TRUFFLE_OBJECT_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), wrappedValues), wrappedValues.length), wrappedValues.length,
1629-
wrappedValues.length,
1630-
ListStorageType.Generic);
1629+
return NativeSequenceStorage.create(callNode.call(FUN_PY_TRUFFLE_OBJECT_ARRAY_TO_NATIVE, wrap(PythonContext.get(callNode), wrappedValues), wrappedValues.length), wrappedValues.length,
1630+
wrappedValues.length, ListStorageType.Generic, true);
16311631
}
16321632

16331633
private static Object wrap(PythonContext context, Object arr) {
@@ -3037,8 +3037,7 @@ private static NativeSequenceStorage reallocNativeSequenceStorage(NativeSequence
30373037
if (lib.isNull(ptr)) {
30383038
throw raiseNode.raise(MemoryError);
30393039
}
3040-
s.setPtr(ptr);
3041-
s.setCapacity(newCapacity);
3040+
s.setPtrAndSize(ptr, newCapacity);
30423041
}
30433042
return s;
30443043
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyMemoryViewFromObject.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ PMemoryView fromManaged(VirtualFrame frame, Object object,
146146
int[] suboffsets = cBuffer.getSuboffsets();
147147
int flags = initFlagsNode.execute(cBuffer.getDims(), cBuffer.getItemSize(), shape, strides, suboffsets);
148148
// TODO when Sulong allows exposing pointers as interop buffer, we can get rid of this
149-
Object pythonBuffer = new NativeSequenceStorage(cBuffer.getBuf(), cBuffer.getLen(), cBuffer.getLen(), SequenceStorage.ListStorageType.Byte);
149+
Object pythonBuffer = NativeSequenceStorage.create(cBuffer.getBuf(), cBuffer.getLen(), cBuffer.getLen(), SequenceStorage.ListStorageType.Byte, false);
150150
TruffleString format = cBuffer.getFormat();
151151
return factory.createMemoryView(PythonContext.get(this), bufferLifecycleManager, pythonBuffer, cBuffer.getObj(), cBuffer.getLen(), cBuffer.isReadOnly(), cBuffer.getItemSize(),
152152
BufferFormat.forMemoryView(format, lengthNode, atIndexNode),

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/NativeSequenceStorage.java

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
package com.oracle.graal.python.runtime.sequence.storage;
4242

4343
import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary;
44+
import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions;
45+
import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.NativeStorageReference;
4446
import com.oracle.truffle.api.CompilerAsserts;
4547
import com.oracle.truffle.api.CompilerDirectives;
4648
import com.oracle.truffle.api.dsl.Cached.Shared;
@@ -60,22 +62,41 @@ public final class NativeSequenceStorage extends SequenceStorage {
6062

6163
private final ListStorageType elementType;
6264

63-
public NativeSequenceStorage(Object ptr, int length, int capacity, ListStorageType elementType) {
65+
private NativeStorageReference reference;
66+
67+
private NativeSequenceStorage(Object ptr, int length, int capacity, ListStorageType elementType) {
6468
super(length, capacity);
6569
this.ptr = ptr;
6670
this.elementType = elementType;
6771
}
6872

73+
/**
74+
* @param ownsMemory whether the memory should be freed when this object dies. Should be true
75+
* when actually used as a sequence storage
76+
*/
77+
public static NativeSequenceStorage create(Object ptr, int length, int capacity, ListStorageType elementType, boolean ownsMemory) {
78+
NativeSequenceStorage storage = new NativeSequenceStorage(ptr, length, capacity, elementType);
79+
if (ownsMemory) {
80+
CApiTransitions.registerNativeSequenceStorage(storage);
81+
}
82+
return storage;
83+
}
84+
6985
public Object getPtr() {
7086
return ptr;
7187
}
7288

73-
public void setPtr(Object ptr) {
89+
public void setPtrAndSize(Object ptr, int size) {
90+
if (reference != null) {
91+
reference.setPtr(ptr);
92+
reference.setSize(size);
93+
}
7494
this.ptr = ptr;
95+
this.capacity = size;
7596
}
7697

77-
public void setCapacity(int capacity) {
78-
this.capacity = capacity;
98+
public void setReference(NativeStorageReference reference) {
99+
this.reference = reference;
79100
}
80101

81102
@Override

0 commit comments

Comments
 (0)