Skip to content

Commit cec0d0d

Browse files
msimacekmsimacek
committed
Implement resizing of native storage
1 parent 3f35a46 commit cec0d0d

File tree

4 files changed

+111
-35
lines changed

4 files changed

+111
-35
lines changed

‎graalpython/com.oracle.graal.python.cext/src/capi.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,12 @@ PyObject* PyTruffle_Object_New(PyTypeObject* cls, PyTypeObject* dominatingNative
511511
} \
512512
return polyglot_from_##__polyglot_type__##_array(carr, len); \
513513
} \
514+
void* PyTruffle_##__jtype__##ArrayRealloc(const void* array, int64_t len) { \
515+
int64_t size = len + 1; \
516+
__ctype__* carr = (__ctype__*) realloc(array, size * sizeof(__ctype__)); \
517+
carr[len] = (__ctype__)0; \
518+
return polyglot_from_##__polyglot_type__##_array(carr, len); \
519+
}
514520

515521
PRIMITIVE_ARRAY_TO_NATIVE(Byte, int8_t, i8, polyglot_as_i8);
516522
PRIMITIVE_ARRAY_TO_NATIVE(Int, int32_t, i32, polyglot_as_i32);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbols.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ public abstract class NativeCAPISymbols {
8282
public static final String FUN_PY_TRUFFLE_LONG_ARRAY_TO_NATIVE = "PyTruffle_LongArrayToNative";
8383
public static final String FUN_PY_TRUFFLE_DOUBLE_ARRAY_TO_NATIVE = "PyTruffle_DoubleArrayToNative";
8484
public static final String FUN_PY_TRUFFLE_OBJECT_ARRAY_TO_NATIVE = "PyTruffle_ObjectArrayToNative";
85+
public static final String FUN_PY_TRUFFLE_BYTE_ARRAY_REALLOC = "PyTruffle_ByteArrayRealloc";
86+
public static final String FUN_PY_TRUFFLE_INT_ARRAY_REALLOC = "PyTruffle_IntArrayRealloc";
87+
public static final String FUN_PY_TRUFFLE_LONG_ARRAY_REALLOC = "PyTruffle_LongArrayRealloc";
88+
public static final String FUN_PY_TRUFFLE_DOUBLE_ARRAY_REALLOC = "PyTruffle_DoubleArrayRealloc";
89+
public static final String FUN_PY_TRUFFLE_OBJECT_ARRAY_REALLOC = "PyTruffle_ObjectArrayRealloc";
8590
public static final String FUN_PY_OBJECT_GENERIC_GET_DICT = "_PyObject_GenericGetDict";
8691
public static final String FUN_PY_OBJECT_NEW = "PyTruffle_Object_New";
8792
public static final String FUN_GET_THREAD_STATE_TYPE_ID = "get_thread_state_typeid";

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/SequenceStorageNodes.java

Lines changed: 97 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,21 @@
2525
*/
2626
package com.oracle.graal.python.builtins.objects.common;
2727

28+
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_BYTE_ARRAY_REALLOC;
2829
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_BYTE_ARRAY_TO_NATIVE;
30+
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_DOUBLE_ARRAY_REALLOC;
2931
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_DOUBLE_ARRAY_TO_NATIVE;
32+
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_INT_ARRAY_REALLOC;
3033
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_INT_ARRAY_TO_NATIVE;
34+
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_LONG_ARRAY_REALLOC;
3135
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_LONG_ARRAY_TO_NATIVE;
36+
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_OBJECT_ARRAY_REALLOC;
3237
import static com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbols.FUN_PY_TRUFFLE_OBJECT_ARRAY_TO_NATIVE;
3338
import static com.oracle.graal.python.nodes.SpecialMethodNames.__EQ__;
3439
import static com.oracle.graal.python.nodes.SpecialMethodNames.__GE__;
3540
import static com.oracle.graal.python.nodes.SpecialMethodNames.__GT__;
3641
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LE__;
3742
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LT__;
38-
import static com.oracle.graal.python.runtime.exception.PythonErrorType.BufferError;
3943
import static com.oracle.graal.python.runtime.exception.PythonErrorType.MemoryError;
4044
import static com.oracle.graal.python.runtime.exception.PythonErrorType.OverflowError;
4145
import static com.oracle.graal.python.runtime.exception.PythonErrorType.SystemError;
@@ -3321,6 +3325,7 @@ public static CreateEmptyNode create() {
33213325
}
33223326

33233327
@GenerateUncached
3328+
@ImportStatic(ListStorageType.class)
33243329
public abstract static class EnsureCapacityNode extends SequenceStorageBaseNode {
33253330

33263331
public abstract SequenceStorage execute(SequenceStorage s, int cap);
@@ -3345,18 +3350,63 @@ static BasicSequenceStorage doManaged(BasicSequenceStorage s, int cap,
33453350
}
33463351
}
33473352

3348-
@Specialization
3349-
static NativeSequenceStorage doNative(@SuppressWarnings("unused") NativeSequenceStorage s, @SuppressWarnings("unused") int cap,
3350-
@Cached PRaiseNode raise) {
3351-
if (s.getElementType() == Byte) {
3352-
// TODO: (mq) need to check for the number of ob_exports, i.e. reference counter.
3353-
// (mq) for the time being we will assume that native storage is exported, i.e.
3354-
// `ob_export > 0`.
3355-
raise.raise(BufferError, ErrorMessages.EXPORTS_CANNOT_RESIZE);
3356-
}
3357-
// TODO re-allocate native memory
3358-
CompilerDirectives.transferToInterpreterAndInvalidate();
3359-
throw new UnsupportedOperationException();
3353+
@Specialization(guards = "s.getElementType() == Byte")
3354+
static NativeSequenceStorage doNativeByte(NativeSequenceStorage s, @SuppressWarnings("unused") int cap,
3355+
@Shared("i") @CachedLibrary(limit = "1") InteropLibrary lib,
3356+
@Exclusive @Cached PCallCapiFunction callCapiFunction,
3357+
@Shared("r") @Cached PRaiseNode raiseNode) {
3358+
return reallocNativeSequenceStorage(s, cap, lib, callCapiFunction, raiseNode, FUN_PY_TRUFFLE_BYTE_ARRAY_REALLOC);
3359+
}
3360+
3361+
@Specialization(guards = "s.getElementType() == Int")
3362+
static NativeSequenceStorage doNativeInt(NativeSequenceStorage s, @SuppressWarnings("unused") int cap,
3363+
@Shared("i") @CachedLibrary(limit = "1") InteropLibrary lib,
3364+
@Exclusive @Cached PCallCapiFunction callCapiFunction,
3365+
@Shared("r") @Cached PRaiseNode raiseNode) {
3366+
return reallocNativeSequenceStorage(s, cap, lib, callCapiFunction, raiseNode, FUN_PY_TRUFFLE_INT_ARRAY_REALLOC);
3367+
}
3368+
3369+
@Specialization(guards = "s.getElementType() == Long")
3370+
static NativeSequenceStorage doNativeLong(NativeSequenceStorage s, @SuppressWarnings("unused") int cap,
3371+
@Shared("i") @CachedLibrary(limit = "1") InteropLibrary lib,
3372+
@Exclusive @Cached PCallCapiFunction callCapiFunction,
3373+
@Shared("r") @Cached PRaiseNode raiseNode) {
3374+
return reallocNativeSequenceStorage(s, cap, lib, callCapiFunction, raiseNode, FUN_PY_TRUFFLE_LONG_ARRAY_REALLOC);
3375+
}
3376+
3377+
@Specialization(guards = "s.getElementType() == Double")
3378+
static NativeSequenceStorage doNativeDouble(NativeSequenceStorage s, @SuppressWarnings("unused") int cap,
3379+
@Shared("i") @CachedLibrary(limit = "1") InteropLibrary lib,
3380+
@Exclusive @Cached PCallCapiFunction callCapiFunction,
3381+
@Shared("r") @Cached PRaiseNode raiseNode) {
3382+
return reallocNativeSequenceStorage(s, cap, lib, callCapiFunction, raiseNode, FUN_PY_TRUFFLE_DOUBLE_ARRAY_REALLOC);
3383+
}
3384+
3385+
@Specialization(guards = "s.getElementType() == Generic")
3386+
static NativeSequenceStorage doNativeObject(NativeSequenceStorage s, @SuppressWarnings("unused") int cap,
3387+
@Shared("i") @CachedLibrary(limit = "1") InteropLibrary lib,
3388+
@Exclusive @Cached PCallCapiFunction callCapiFunction,
3389+
@Shared("r") @Cached PRaiseNode raiseNode) {
3390+
return reallocNativeSequenceStorage(s, cap, lib, callCapiFunction, raiseNode, FUN_PY_TRUFFLE_OBJECT_ARRAY_REALLOC);
3391+
}
3392+
3393+
private static NativeSequenceStorage reallocNativeSequenceStorage(NativeSequenceStorage s, int requestedCapacity, InteropLibrary lib, PCallCapiFunction callCapiFunction, PRaiseNode raiseNode,
3394+
String function) {
3395+
if (requestedCapacity > s.getCapacity()) {
3396+
int newCapacity;
3397+
try {
3398+
newCapacity = Math.max(16, PythonUtils.multiplyExact(requestedCapacity, 2));
3399+
} catch (OverflowException e) {
3400+
newCapacity = requestedCapacity;
3401+
}
3402+
Object ptr = callCapiFunction.call(function, s.getPtr(), newCapacity);
3403+
if (lib.isNull(ptr)) {
3404+
throw raiseNode.raise(MemoryError);
3405+
}
3406+
s.setPtr(ptr);
3407+
s.setCapacity(newCapacity);
3408+
}
3409+
return s;
33603410
}
33613411
}
33623412

@@ -3543,22 +3593,22 @@ public DeleteNode(NormalizeIndexNode normalizeIndexNode, String keyTypeErrorMess
35433593

35443594
@Specialization
35453595
protected void doScalarInt(VirtualFrame frame, SequenceStorage storage, int idx) {
3546-
getGetItemScalarNode().execute(storage, normalizeIndex(frame, idx, storage));
3596+
getDeleteItemNode().execute(storage, normalizeIndex(frame, idx, storage));
35473597
}
35483598

35493599
@Specialization
35503600
protected void doScalarLong(VirtualFrame frame, SequenceStorage storage, long idx) {
3551-
getGetItemScalarNode().execute(storage, normalizeIndex(frame, idx, storage));
3601+
getDeleteItemNode().execute(storage, normalizeIndex(frame, idx, storage));
35523602
}
35533603

35543604
@Specialization
35553605
protected void doScalarPInt(VirtualFrame frame, SequenceStorage storage, PInt idx) {
3556-
getGetItemScalarNode().execute(storage, normalizeIndex(frame, idx, storage));
3606+
getDeleteItemNode().execute(storage, normalizeIndex(frame, idx, storage));
35573607
}
35583608

35593609
@Specialization(guards = "!isPSlice(idx)")
35603610
protected void doScalarGeneric(VirtualFrame frame, SequenceStorage storage, Object idx) {
3561-
getGetItemScalarNode().execute(storage, normalizeIndex(frame, idx, storage));
3611+
getDeleteItemNode().execute(storage, normalizeIndex(frame, idx, storage));
35623612
}
35633613

35643614
@Specialization
@@ -3571,7 +3621,7 @@ protected void doSlice(SequenceStorage storage, PSlice slice,
35713621
SliceInfo unadjusted = unpack.execute(sliceCast.execute(slice));
35723622
SliceInfo info = adjustIndices.execute(len, unadjusted);
35733623
try {
3574-
getGetItemSliceNode().execute(storage, info);
3624+
getDeleteSliceNode().execute(storage, info);
35753625
} catch (SequenceStoreException e) {
35763626
CompilerDirectives.transferToInterpreterAndInvalidate();
35773627
throw new IllegalStateException();
@@ -3587,15 +3637,15 @@ protected void doInvalidKey(@SuppressWarnings("unused") SequenceStorage storage,
35873637
throw raiseNode.raise(TypeError, keyTypeErrorMessage, key);
35883638
}
35893639

3590-
private DeleteItemNode getGetItemScalarNode() {
3640+
private DeleteItemNode getDeleteItemNode() {
35913641
if (deleteItemNode == null) {
35923642
CompilerDirectives.transferToInterpreterAndInvalidate();
35933643
deleteItemNode = insert(DeleteItemNode.create());
35943644
}
35953645
return deleteItemNode;
35963646
}
35973647

3598-
private DeleteSliceNode getGetItemSliceNode() {
3648+
private DeleteSliceNode getDeleteSliceNode() {
35993649
if (deleteSliceNode == null) {
36003650
CompilerDirectives.transferToInterpreterAndInvalidate();
36013651
deleteSliceNode = insert(DeleteSliceNode.create());
@@ -3993,23 +4043,47 @@ public final SequenceStorage execute(SequenceStorage storage, int index, Object
39934043

39944044
protected abstract SequenceStorage execute(SequenceStorage storage, int index, Object value, boolean recursive);
39954045

4046+
@Specialization
4047+
protected static SequenceStorage doStorage(EmptySequenceStorage storage, int index, Object value, boolean recursive,
4048+
@Cached InsertItemNode recursiveNode) {
4049+
if (!recursive) {
4050+
throw CompilerDirectives.shouldNotReachHere();
4051+
}
4052+
SequenceStorage newStorage = storage.generalizeFor(value, null);
4053+
return recursiveNode.execute(newStorage, index, value, false);
4054+
}
4055+
39964056
@Specialization(limit = "MAX_ARRAY_STORAGES", guards = {"storage.getClass() == cachedClass"})
3997-
protected static SequenceStorage doStorage(SequenceStorage storage, int index, Object value, boolean recursive,
4057+
protected static SequenceStorage doStorage(BasicSequenceStorage storage, int index, Object value, boolean recursive,
39984058
@Cached InsertItemNode recursiveNode,
39994059
@Cached("storage.getClass()") Class<? extends SequenceStorage> cachedClass) {
40004060
try {
40014061
cachedClass.cast(storage).insertItem(index, value);
40024062
return storage;
40034063
} catch (SequenceStoreException e) {
40044064
if (!recursive) {
4005-
CompilerDirectives.transferToInterpreterAndInvalidate();
4006-
throw new IllegalStateException();
4065+
throw CompilerDirectives.shouldNotReachHere();
40074066
}
40084067
SequenceStorage newStorage = cachedClass.cast(storage).generalizeFor(value, null);
40094068
return recursiveNode.execute(newStorage, index, value, false);
40104069
}
40114070
}
40124071

4072+
@Specialization
4073+
protected static SequenceStorage doStorage(NativeSequenceStorage storage, int index, Object value, @SuppressWarnings("unused") boolean recursive,
4074+
@Cached EnsureCapacityNode ensureCapacityNode,
4075+
@Cached GetItemScalarNode getItem,
4076+
@Cached SetItemScalarNode setItem) {
4077+
int newLength = storage.length() + 1;
4078+
ensureCapacityNode.execute(storage, newLength);
4079+
for (int i = storage.length(); i > index; i--) {
4080+
setItem.execute(storage, i, getItem.execute(storage, i - 1));
4081+
}
4082+
setItem.execute(storage, index, value);
4083+
storage.setNewLength(newLength);
4084+
return storage;
4085+
}
4086+
40134087
public static InsertItemNode create() {
40144088
return InsertItemNodeGen.create();
40154089
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/NativeSequenceStorage.java

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@
4040
*/
4141
package com.oracle.graal.python.runtime.sequence.storage;
4242

43-
import com.oracle.graal.python.PythonLanguage;
44-
import com.oracle.graal.python.nodes.ErrorMessages;
45-
import com.oracle.graal.python.runtime.exception.PythonErrorType;
4643
import com.oracle.truffle.api.CompilerAsserts;
4744
import com.oracle.truffle.api.CompilerDirectives;
4845

@@ -104,16 +101,10 @@ public String toString() {
104101
return String.format("<NativeSequenceStorage(type=%s, len=%d, cap=%d) at %s>", elementType, len, capacity, ptr);
105102
}
106103

107-
/**
108-
* Ensure that the current capacity is big enough. If not, we increase capacity to the next
109-
* designated size (not necessarily the requested one).
110-
*/
111104
@Override
112-
public void ensureCapacity(int newCapacity) {
113-
if (newCapacity > capacity) {
114-
CompilerDirectives.transferToInterpreterAndInvalidate();
115-
throw PythonLanguage.getCore().raise(PythonErrorType.BufferError, ErrorMessages.CANNOT_RESIZE_BUFFER);
116-
}
105+
public void ensureCapacity(@SuppressWarnings("unused") int newCapacity) {
106+
CompilerDirectives.transferToInterpreterAndInvalidate();
107+
throw new IllegalStateException("should not reach");
117108
}
118109

119110
@Override

0 commit comments

Comments
 (0)