Skip to content

Commit 72a756a

Browse files
horakivohorakivo
committed
[GR-48910]: Arrow storage strategy
PullRequest: graalpython/3369
2 parents d647e07 + 22e684d commit 72a756a

File tree

18 files changed

+673
-18
lines changed

18 files changed

+673
-18
lines changed

‎ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ "overlay": "688bc4f7e7a246cd2bc332a19eff836b01f84c17" }
1+
{ "overlay": "68d2386ceb05a9094862df57e8912a1207499e89" }

‎graalpython/com.oracle.graal.python.test/src/tests/util.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,4 @@
4343
def storage_to_native(s):
4444
if sys.implementation.name == 'graalpy':
4545
assert hasattr(__graalpython__, 'storage_to_native'), "Needs to be run with --python.EnableDebuggingBuiltins"
46-
__graalpython__.storage_to_native(s)
46+
__graalpython__.storage_to_native(s)

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@
7878
import java.util.List;
7979
import java.util.logging.Level;
8080

81+
import com.oracle.graal.python.nodes.util.ToNativePrimitiveStorageNode;
82+
import com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage;
8183
import org.graalvm.home.Version;
8284
import org.graalvm.nativeimage.ImageInfo;
8385

@@ -163,6 +165,7 @@
163165
import com.oracle.truffle.api.TruffleLogger;
164166
import com.oracle.truffle.api.dsl.Bind;
165167
import com.oracle.truffle.api.dsl.Cached;
168+
import com.oracle.truffle.api.dsl.Cached.Shared;
166169
import com.oracle.truffle.api.dsl.Fallback;
167170
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
168171
import com.oracle.truffle.api.dsl.NeverDefault;
@@ -1087,4 +1090,27 @@ TruffleString get() {
10871090
return context.getStdlibHome().concatUncached(sep, TS_ENCODING, false).concatUncached(context.getCoreHome(), TS_ENCODING, false);
10881091
}
10891092
}
1093+
1094+
@Builtin(name = "storage_to_native_primitive", minNumOfPositionalArgs = 1)
1095+
@GenerateNodeFactory
1096+
abstract static class StorageToNativePrimitive extends PythonUnaryBuiltinNode {
1097+
1098+
@Specialization
1099+
static Object doArray(PArray array,
1100+
@Shared @Cached ToNativePrimitiveStorageNode toNativePrimitiveNode,
1101+
@Bind("this") Node inliningTarget) {
1102+
NativePrimitiveSequenceStorage newStorage = toNativePrimitiveNode.execute(inliningTarget, array.getSequenceStorage());
1103+
array.setSequenceStorage(newStorage);
1104+
return array;
1105+
}
1106+
1107+
@Specialization
1108+
static Object doSequence(PSequence sequence,
1109+
@Shared @Cached ToNativePrimitiveStorageNode toNativePrimitiveNode,
1110+
@Bind("this") Node inliningTarget) {
1111+
NativePrimitiveSequenceStorage newStorage = toNativePrimitiveNode.execute(inliningTarget, sequence.getSequenceStorage());
1112+
sequence.setSequenceStorage(newStorage);
1113+
return sequence;
1114+
}
1115+
}
10901116
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesCommonBuiltins.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,7 @@ abstract static class PrefixSuffixBaseNode extends PythonQuaternaryClinicBuiltin
446446
// common and specialized cases --------------------
447447

448448
@Specialization
449+
@SuppressWarnings("truffle-static-method")
449450
boolean doIt(VirtualFrame frame, Object self, Object substrs, int start, int end,
450451
@Bind("this") Node inliningTarget,
451452
@Cached GetBytesStorage getBytesStorage,

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PySequenceArrayWrapper.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
import java.util.logging.Level;
4444

45+
import com.oracle.graal.python.PythonLanguage;
4546
import com.oracle.graal.python.builtins.objects.bytes.PBytesLike;
4647
import com.oracle.graal.python.builtins.objects.cext.capi.PySequenceArrayWrapperFactory.ToNativeStorageNodeGen;
4748
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
@@ -54,6 +55,7 @@
5455
import com.oracle.graal.python.runtime.sequence.storage.MroSequenceStorage;
5556
import com.oracle.graal.python.runtime.sequence.storage.NativeSequenceStorage;
5657
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
58+
import com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage;
5759
import com.oracle.graal.python.util.PythonUtils;
5860
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
5961
import com.oracle.truffle.api.TruffleLogger;
@@ -77,6 +79,8 @@ public final class PySequenceArrayWrapper {
7779
@GenerateUncached
7880
public abstract static class ToNativeStorageNode extends Node {
7981

82+
private static final TruffleLogger LOGGER = PythonLanguage.getLogger(ToNativeStorageNode.class);
83+
8084
public abstract NativeSequenceStorage execute(Node inliningTarget, SequenceStorage object, boolean isBytesLike);
8185

8286
public static NativeSequenceStorage executeUncached(SequenceStorage object, boolean isBytesLike) {
@@ -86,7 +90,7 @@ public static NativeSequenceStorage executeUncached(SequenceStorage object, bool
8690
@Specialization(guards = "!isMroSequenceStorage(s)")
8791
static NativeSequenceStorage doManaged(Node inliningTarget, ArrayBasedSequenceStorage s, boolean isBytesLike,
8892
@Exclusive @Cached SequenceStorageNodes.StorageToNativeNode storageToNativeNode,
89-
@Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
93+
@Exclusive @Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
9094
Object array;
9195
if (isBytesLike) {
9296
ByteSequenceStorage byteStorage = (ByteSequenceStorage) s;
@@ -97,6 +101,18 @@ static NativeSequenceStorage doManaged(Node inliningTarget, ArrayBasedSequenceSt
97101
return storageToNativeNode.execute(inliningTarget, array, s.length());
98102
}
99103

104+
/*
105+
* TODO This can be optimized further. Now we are converting NativePrimitiveSequenceStorage
106+
* to ObjectArrayStorage and then to NativeStorageStrategy
107+
*/
108+
@Specialization
109+
static NativeSequenceStorage doNativePrimitive(Node inliningTarget, NativePrimitiveSequenceStorage s, boolean isBytesLike,
110+
@Exclusive @Cached SequenceStorageNodes.StorageToNativeNode storageToNativeNode,
111+
@Exclusive @Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
112+
Object array = getInternalArrayNode.execute(inliningTarget, s);
113+
return storageToNativeNode.execute(inliningTarget, array, s.length());
114+
}
115+
100116
/*
101117
* This specialization uses a TruffleBoundary because we assume that there is a fixed number
102118
* of types (and therefore MroSequenceStorages). If types are created on a fast path, this

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/SequenceStorageNodes.java

Lines changed: 104 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,10 @@
109109
import com.oracle.graal.python.nodes.object.GetClassNode.GetPythonObjectClassNode;
110110
import com.oracle.graal.python.nodes.util.CastToByteNode;
111111
import com.oracle.graal.python.nodes.util.CastToJavaByteNode;
112+
import com.oracle.graal.python.runtime.PythonContext;
112113
import com.oracle.graal.python.runtime.PythonOptions;
113114
import com.oracle.graal.python.runtime.exception.PException;
115+
import com.oracle.graal.python.runtime.native_memory.NativeBuffer;
114116
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
115117
import com.oracle.graal.python.runtime.sequence.PSequence;
116118
import com.oracle.graal.python.runtime.sequence.storage.ArrayBasedSequenceStorage;
@@ -129,6 +131,8 @@
129131
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage.StorageType;
130132
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorageFactory;
131133
import com.oracle.graal.python.runtime.sequence.storage.SequenceStoreException;
134+
import com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage;
135+
import com.oracle.graal.python.runtime.sequence.storage.NativeIntSequenceStorage;
132136
import com.oracle.graal.python.util.BiFunction;
133137
import com.oracle.graal.python.util.OverflowException;
134138
import com.oracle.graal.python.util.PythonUtils;
@@ -163,6 +167,7 @@
163167
import com.oracle.truffle.api.profiles.InlinedExactClassProfile;
164168
import com.oracle.truffle.api.profiles.InlinedLoopConditionProfile;
165169
import com.oracle.truffle.api.strings.TruffleString;
170+
import sun.misc.Unsafe;
166171

167172
public abstract class SequenceStorageNodes {
168173

@@ -610,6 +615,11 @@ protected static Object doObject(ObjectSequenceStorage storage, int idx) {
610615
return storage.getObjectItemNormalized(idx);
611616
}
612617

618+
@Specialization
619+
protected static int doNativeInt(NativeIntSequenceStorage storage, int idx) {
620+
return storage.getIntItemNormalized(idx);
621+
}
622+
613623
@Specialization
614624
protected static Object doMro(MroSequenceStorage storage, int idx) {
615625
return storage.getPythonClassItemNormalized(idx);
@@ -775,6 +785,13 @@ protected static SequenceStorage doMroSequenceStorage(MroSequenceStorage storage
775785
return new ObjectSequenceStorage(newArray);
776786
}
777787

788+
@Specialization
789+
protected static SequenceStorage doNativeInt(NativeIntSequenceStorage storage, int start, int stop, int step, int length,
790+
@Bind("this") Node node) {
791+
NativeBuffer sliceValueBuffer = doNativePrimitiveSliceInBound(PythonContext.get(node), start, step, length, storage);
792+
return PythonContext.get(node).nativeBufferContext.createNativeIntStorage(sliceValueBuffer, length);
793+
}
794+
778795
@Specialization
779796
protected static SequenceStorage doNativeByte(NativeByteSequenceStorage storage, int start, @SuppressWarnings("unused") int stop, int step, int length,
780797
@Cached CStructAccess.ReadByteNode readNode) {
@@ -797,6 +814,27 @@ protected static SequenceStorage doNativeObject(NativeObjectSequenceStorage stor
797814
return new ObjectSequenceStorage(newArray);
798815
}
799816

817+
private static NativeBuffer doNativePrimitiveSliceInBound(PythonContext pythonCtx, int start, int step, int sliceLength, NativePrimitiveSequenceStorage storage) {
818+
var unsafe = pythonCtx.getUnsafe();
819+
long itemSize = storage.getItemSize();
820+
long sizeInBytes = sliceLength * itemSize;
821+
NativeBuffer sliceBuffer = NativeBuffer.allocateNew(sizeInBytes);
822+
823+
if (step == 1) {
824+
var startAddress = storage.getValueBufferAddr() + (start * itemSize);
825+
unsafe.copyMemory(startAddress, sliceBuffer.getMemoryAddress(), sizeInBytes);
826+
return sliceBuffer;
827+
}
828+
829+
var stepInBytes = step * itemSize;
830+
for (long srcAddr = storage.getValueBufferAddr() + (start * itemSize), destAddr = sliceBuffer.getMemoryAddress(),
831+
j = 0; j < sliceLength; srcAddr += stepInBytes, destAddr += itemSize, j++) {
832+
unsafe.copyMemory(srcAddr, destAddr, itemSize);
833+
}
834+
835+
return sliceBuffer;
836+
}
837+
800838
@NeverDefault
801839
public static GetItemSliceNode create() {
802840
return GetItemSliceNodeGen.create();
@@ -1183,6 +1221,11 @@ protected static void doInt(@SuppressWarnings("unused") Node inliningTarget, Int
11831221
storage.setIntItemNormalized(idx, value);
11841222
}
11851223

1224+
@Specialization
1225+
protected static void doNativeInt(@SuppressWarnings("unused") Node inliningTarget, NativeIntSequenceStorage storage, int idx, int value) {
1226+
storage.setIntItemNormalized(idx, value);
1227+
}
1228+
11861229
@Specialization(rewriteOn = OverflowException.class)
11871230
protected static void doIntL(@SuppressWarnings("unused") Node inliningTarget, IntSequenceStorage storage, int idx, long value) throws OverflowException {
11881231
storage.setIntItemNormalized(idx, PInt.intValueExact(value));
@@ -1515,6 +1558,23 @@ static void doObjectStorage(ObjectSequenceStorage storage) {
15151558
storage.reverse();
15161559
}
15171560

1561+
@Specialization
1562+
static void doNativePrimitive(Node inliningTarget, NativePrimitiveSequenceStorage storage) {
1563+
var length = storage.length();
1564+
var unsafe = PythonContext.get(inliningTarget).getUnsafe();
1565+
long itemSize = storage.getItemSize();
1566+
long startAddress = storage.getValueBufferAddr();
1567+
long endAddress = startAddress + ((length - 1) * itemSize);
1568+
byte[] tempBuffer = new byte[(int) itemSize];
1569+
while (startAddress < endAddress) {
1570+
unsafe.copyMemory(null, startAddress, tempBuffer, Unsafe.ARRAY_BYTE_BASE_OFFSET, itemSize);
1571+
unsafe.copyMemory(endAddress, startAddress, itemSize);
1572+
unsafe.copyMemory(tempBuffer, Unsafe.ARRAY_BYTE_BASE_OFFSET, null, endAddress, itemSize);
1573+
startAddress += itemSize;
1574+
endAddress -= itemSize;
1575+
}
1576+
}
1577+
15181578
@Specialization
15191579
static void doBoolStorage(BoolSequenceStorage storage) {
15201580
storage.reverse();
@@ -3083,6 +3143,13 @@ static void doObject(ObjectSequenceStorage storage, int cap) {
30833143
storage.ensureCapacity(cap);
30843144
}
30853145

3146+
@Specialization
3147+
static void doNativePrimitive(NativePrimitiveSequenceStorage storage, int cap) {
3148+
if (CompilerDirectives.injectBranchProbability(CompilerDirectives.UNLIKELY_PROBABILITY, cap > storage.getCapacity())) {
3149+
storage.reallocate(cap);
3150+
}
3151+
}
3152+
30863153
@Specialization
30873154
static void doBool(BoolSequenceStorage storage, int cap) {
30883155
storage.ensureCapacity(cap);
@@ -3177,7 +3244,7 @@ private static int computeNewCapacity(int cap) {
31773244
@GenerateCached(false)
31783245
public abstract static class CopyNode extends Node {
31793246

3180-
public abstract SequenceStorage execute(Node node, SequenceStorage s);
3247+
public abstract SequenceStorage execute(Node inliningTarget, SequenceStorage s);
31813248

31823249
public static SequenceStorage executeUncached(SequenceStorage s) {
31833250
return SequenceStorageNodesFactory.CopyNodeGen.getUncached().execute(null, s);
@@ -3223,6 +3290,14 @@ static SequenceStorage doMro(MroSequenceStorage storage) {
32233290
return new ObjectSequenceStorage(PythonUtils.arrayCopyOf(storage.getInternalClassArray(), storage.length()));
32243291
}
32253292

3293+
@Specialization
3294+
static SequenceStorage doNativeInt(Node inliningTarget, NativeIntSequenceStorage storage) {
3295+
var nativeContext = PythonContext.get(inliningTarget).getContext().nativeBufferContext;
3296+
var copiedBuffer = storage.getValueBuffer().copy();
3297+
3298+
return nativeContext.createNativeIntStorage(copiedBuffer, storage.length());
3299+
}
3300+
32263301
@Specialization
32273302
static SequenceStorage doNativeBytes(NativeByteSequenceStorage s,
32283303
@Shared @Cached(inline = false) GetNativeItemScalarNode getItem) {
@@ -3330,6 +3405,11 @@ static void doNative(NativeSequenceStorage s, int len,
33303405
@Cached(inline = false) SetNativeLenNode setLen) {
33313406
setLen.execute(s, len);
33323407
}
3408+
3409+
@Specialization
3410+
static void doNativePrimitive(NativePrimitiveSequenceStorage s, int len) {
3411+
s.setNewLength(len);
3412+
}
33333413
}
33343414

33353415
@GenerateUncached
@@ -3441,15 +3521,15 @@ public abstract static class DeleteItemNode extends SequenceStorageBaseNode {
34413521

34423522
@Specialization(guards = "isLastItem(s, idx)")
34433523
static void doLastItem(Node inliningTarget, SequenceStorage s, @SuppressWarnings("unused") int idx,
3444-
@Shared @Cached SetLenNode setLenNode) {
3524+
@Exclusive @Cached SetLenNode setLenNode) {
34453525
setLenNode.execute(inliningTarget, s, s.length() - 1);
34463526
}
34473527

34483528
@Specialization
34493529
static void doGeneric(Node inliningTarget, SequenceStorage s, int idx,
34503530
@Cached GetItemScalarNode getItemNode,
34513531
@Cached SetItemScalarNode setItemNode,
3452-
@Shared @Cached SetLenNode setLenNode) {
3532+
@Exclusive @Cached SetLenNode setLenNode) {
34533533
int len = s.length();
34543534

34553535
for (int i = idx; i < len - 1; i++) {
@@ -3860,9 +3940,29 @@ static SequenceStorage doArrayBasedStorage(Node inliningTarget, ArrayBasedSequen
38603940

38613941
}
38623942

3943+
// TODO introduce something similar to InsertItemArrayBasedStorageNode
3944+
@Specialization
3945+
static SequenceStorage doNativeInt(Node inliningTarget, NativeIntSequenceStorage storage, int index, int value,
3946+
@Exclusive @Cached EnsureCapacityNode ensureCapacity) {
3947+
int length = storage.length();
3948+
var context = PythonContext.get(inliningTarget);
3949+
var unsafe = context.getUnsafe();
3950+
long itemSize = storage.getItemSize();
3951+
ensureCapacity.execute(inliningTarget, storage, length + 1);
3952+
// shifting tail to the right by one slot
3953+
long startAddr = storage.getValueBufferAddr() + (index * itemSize);
3954+
long endAddr = startAddr + itemSize;
3955+
long sizeInBytes = (length - index) * itemSize;
3956+
unsafe.copyMemory(startAddr, endAddr, sizeInBytes);
3957+
3958+
storage.setIntItemNormalized(index, value);
3959+
storage.incLength();
3960+
return storage;
3961+
}
3962+
38633963
@Specialization
38643964
protected static SequenceStorage doNativeStorage(Node inliningTarget, NativeSequenceStorage storage, int index, Object value,
3865-
@Cached EnsureCapacityNode ensureCapacityNode,
3965+
@Exclusive @Cached EnsureCapacityNode ensureCapacityNode,
38663966
@Cached(inline = false) GetItemScalarNode getItem,
38673967
@Cached SetItemScalarNode setItem) {
38683968
int newLength = storage.length() + 1;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PNodeWithContext.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ public final PythonContext getContext() {
8181
return PythonContext.get(this);
8282
}
8383

84+
@NonIdempotent
85+
public static PythonContext getContext(Node node) {
86+
return PythonContext.get(node);
87+
}
88+
8489
@Idempotent
8590
public final boolean isSingleContext() {
8691
return getLanguage().isSingleContext();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5035,13 +5035,19 @@ private void bytecodeLoadConstCollection(VirtualFrame virtualFrame, int stackTop
50355035
int kind = CollectionBits.collectionKind(typeAndKind);
50365036
assert kind == CollectionBits.KIND_LIST || kind == CollectionBits.KIND_TUPLE;
50375037
boolean list = kind == CollectionBits.KIND_LIST;
5038+
var context = PythonContext.get(this);
5039+
boolean useNativePrimitiveStorage = context.getLanguage().getEngineOption(PythonOptions.UseNativePrimitiveStorageStrategy);
50385040
switch (CollectionBits.elementType(typeAndKind)) {
50395041
case CollectionBits.ELEMENT_INT: {
50405042
int[] a = (int[]) array;
5041-
if (list) {
5042-
a = PythonUtils.arrayCopyOf(a, a.length);
5043+
if (useNativePrimitiveStorage) {
5044+
storage = context.nativeBufferContext.toNativeIntStorage(a);
5045+
} else {
5046+
if (list) {
5047+
a = PythonUtils.arrayCopyOf(a, a.length);
5048+
}
5049+
storage = new IntSequenceStorage(a);
50435050
}
5044-
storage = new IntSequenceStorage(a);
50455051
break;
50465052
}
50475053
case CollectionBits.ELEMENT_LONG: {

0 commit comments

Comments
 (0)