Skip to content

Commit ce8c16b

Browse files
horakivohorakivo
committed
Add python unittests task
1 parent 25ecc66 commit ce8c16b

File tree

9 files changed

+79
-88
lines changed

9 files changed

+79
-88
lines changed

‎ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ "overlay": "688bc4f7e7a246cd2bc332a19eff836b01f84c17" }
1+
{ "overlay": "8f5db1c15647ade11013e2ed88a39d4e59bdab72" }

‎graalpython/com.oracle.graal.python.test/src/tests/test_arrow_storage_array.py

Lines changed: 0 additions & 64 deletions
This file was deleted.

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PySequenceArrayWrapper.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
import java.util.logging.Level;
4444

45+
import com.oracle.graal.python.PythonLanguage;
4546
import com.oracle.graal.python.builtins.objects.bytes.PBytesLike;
4647
import com.oracle.graal.python.builtins.objects.cext.capi.PySequenceArrayWrapperFactory.ToNativeStorageNodeGen;
4748
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
@@ -78,18 +79,18 @@ public final class PySequenceArrayWrapper {
7879
@GenerateUncached
7980
public abstract static class ToNativeStorageNode extends Node {
8081

82+
private static final TruffleLogger LOGGER = PythonLanguage.getLogger(ToNativeStorageNode.class);
83+
8184
public abstract NativeSequenceStorage execute(Node inliningTarget, SequenceStorage object, boolean isBytesLike);
8285

8386
public static NativeSequenceStorage executeUncached(SequenceStorage object, boolean isBytesLike) {
8487
return ToNativeStorageNodeGen.getUncached().execute(null, object, isBytesLike);
8588
}
8689

87-
// TODO Ivo REMOVE warning
8890
@Specialization(guards = "!isMroSequenceStorage(s)")
89-
@SuppressWarnings("truffle-sharing")
9091
static NativeSequenceStorage doManaged(Node inliningTarget, ArrayBasedSequenceStorage s, boolean isBytesLike,
9192
@Exclusive @Cached SequenceStorageNodes.StorageToNativeNode storageToNativeNode,
92-
@Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
93+
@Exclusive @Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
9394
Object array;
9495
if (isBytesLike) {
9596
ByteSequenceStorage byteStorage = (ByteSequenceStorage) s;
@@ -100,12 +101,11 @@ static NativeSequenceStorage doManaged(Node inliningTarget, ArrayBasedSequenceSt
100101
return storageToNativeNode.execute(inliningTarget, array, s.length());
101102
}
102103

103-
// TODO Ivo TEMPORAL
104104
@Specialization
105-
@SuppressWarnings("truffle-sharing")
106105
static NativeSequenceStorage doArrow(Node inliningTarget, ArrowSequenceStorage s, boolean isBytesLike,
107106
@Exclusive @Cached SequenceStorageNodes.StorageToNativeNode storageToNativeNode,
108-
@Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
107+
@Exclusive @Cached SequenceStorageNodes.GetInternalObjectArrayNode getInternalArrayNode) {
108+
LOGGER.warning("The sequence backed by Arrow Storage is being converted to the Native Storage strategy. This operation is slow and should not typically occur.");
109109
Object array = getInternalArrayNode.execute(inliningTarget, s);
110110
return storageToNativeNode.execute(inliningTarget, array, s.length());
111111
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@
208208
import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage;
209209
import com.oracle.graal.python.runtime.sequence.storage.ObjectSequenceStorage;
210210
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
211+
import com.oracle.graal.python.runtime.sequence.storage.native2.IntArrowSequenceStorage;
211212
import com.oracle.graal.python.util.PythonUtils;
212213
import com.oracle.truffle.api.Assumption;
213214
import com.oracle.truffle.api.CompilerAsserts;
@@ -5035,13 +5036,20 @@ private void bytecodeLoadConstCollection(VirtualFrame virtualFrame, int stackTop
50355036
int kind = CollectionBits.collectionKind(typeAndKind);
50365037
assert kind == CollectionBits.KIND_LIST || kind == CollectionBits.KIND_TUPLE;
50375038
boolean list = kind == CollectionBits.KIND_LIST;
5039+
var context = PythonContext.get(this);
5040+
boolean useNativeStorageStrategy = context.getLanguage().getEngineOption(PythonOptions.UseNativeStorageStrategy);
50385041
switch (CollectionBits.elementType(typeAndKind)) {
50395042
case CollectionBits.ELEMENT_INT: {
50405043
int[] a = (int[]) array;
5041-
if (list) {
5042-
a = PythonUtils.arrayCopyOf(a, a.length);
5044+
if (useNativeStorageStrategy) {
5045+
var nativeBuffer = context.nativeBufferContext.toNativeBuffer(a);
5046+
storage = new IntArrowSequenceStorage(nativeBuffer, a.length);
5047+
} else {
5048+
if (list) {
5049+
a = PythonUtils.arrayCopyOf(a, a.length);
5050+
}
5051+
storage = new IntSequenceStorage(a);
50435052
}
5044-
storage = new IntSequenceStorage(a);
50455053
break;
50465054
}
50475055
case CollectionBits.ELEMENT_LONG: {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,9 @@ private PythonOptions() {
382382
@Option(category = OptionCategory.EXPERT, help = "If true, use the system's toolchain for native extension compilation. Otherwise, use the LLVM Toolchain included with GraalVM.") //
383383
public static final OptionKey<Boolean> UseSystemToolchain = new OptionKey<>(true);
384384

385+
@EngineOption @Option(category = OptionCategory.INTERNAL, usageSyntax = "true|false", help = "If true, uses native storage strategy for primitive types") //
386+
public static final OptionKey<Boolean> UseNativeStorageStrategy = new OptionKey<>(false);
387+
385388
public static final OptionDescriptors DESCRIPTORS = new PythonOptionsOptionDescriptors();
386389

387390
@CompilationFinal(dimensions = 1) private static final OptionKey<?>[] ENGINE_OPTION_KEYS;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/native2/IntArrowSequenceStorage.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242

4343
public class IntArrowSequenceStorage extends ArrowSequenceStorage {
4444

45-
private static final long TYPE_WIDTH = Integer.BYTES;
45+
private static final byte TYPE_WIDTH = Integer.BYTES;
4646

4747
public IntArrowSequenceStorage(NativeBuffer nativeBuffer, int length) {
4848
super(nativeBuffer, length, TYPE_WIDTH);
@@ -59,12 +59,12 @@ public Object getIndicativeValue() {
5959
}
6060

6161
public int getIntItemNormalized(int idx) {
62-
long indexInBytes = idx * TYPE_WIDTH;
62+
long indexInBytes = (long) idx * (long) TYPE_WIDTH;
6363
return nativeBuffer.getInt(indexInBytes);
6464
}
6565

6666
public void setIntItemNormalized(int idx, int value) {
67-
long indexInBytes = idx * TYPE_WIDTH;
67+
long indexInBytes = (long) idx * (long) TYPE_WIDTH;
6868
nativeBuffer.setInt(indexInBytes, value);
6969
}
7070
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/native2/NativeBufferDeallocatorRunnable.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,19 +63,19 @@ public NativeBufferDeallocatorRunnable(ReferenceQueue<NativeBuffer> referenceQue
6363

6464
@Override
6565
public void run() {
66-
try {
67-
PythonContext pythonContext = PythonContext.get(null);
68-
PythonLanguage language = pythonContext.getLanguage();
66+
PythonContext pythonContext = PythonContext.get(null);
67+
PythonLanguage language = pythonContext.getLanguage();
6968

70-
while (!pythonContext.getThreadState(language).isShuttingDown()) {
69+
while (!pythonContext.getThreadState(language).isShuttingDown()) {
70+
try {
7171
NativeBufferReference phantomRef = (NativeBufferReference) referenceQueue.remove();
72-
if (phantomRef != null) {
73-
unsafe.freeMemory(phantomRef.getMemoryAddress());
74-
references.remove(phantomRef);
75-
}
72+
unsafe.freeMemory(phantomRef.getMemoryAddress());
73+
references.remove(phantomRef);
74+
} catch (InterruptedException e) {
75+
Thread.currentThread().interrupt();
76+
return;
7677
}
77-
} catch (PythonThreadKillException | InterruptedException e) {
78-
// TODO log it here
7978
}
79+
8080
}
8181
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/native2/ToArrowStorageNode.java

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,46 @@
1+
/*
2+
* Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
141
package com.oracle.graal.python.runtime.sequence.storage.native2;
242

343
import com.oracle.graal.python.runtime.PythonContext;
4-
import com.oracle.graal.python.runtime.sequence.storage.ArrayBasedSequenceStorage;
544
import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage;
645
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
746
import com.oracle.truffle.api.dsl.GenerateCached;

‎mx.graalpython/mx_graalpython.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,7 @@ class GraalPythonTags(object):
760760
unittest_sandboxed = 'python-unittest-sandboxed'
761761
unittest_multi = 'python-unittest-multi-context'
762762
unittest_jython = 'python-unittest-jython'
763+
unittest_arrow = 'python-unittest-arrow-storage'
763764
unittest_hpy = 'python-unittest-hpy'
764765
unittest_hpy_sandboxed = 'python-unittest-hpy-sandboxed'
765766
unittest_posix = 'python-unittest-posix'
@@ -1487,6 +1488,10 @@ def graalpython_gate_runner(args, tasks):
14871488
if task:
14881489
run_python_unittests(graalpy_standalone_jvm(), args=["--python.EmulateJython"], paths=["test_interop.py"], report=report(), nonZeroIsFatal=nonZeroIsFatal)
14891490

1491+
with Task('GraalPython with Arrow Storage Strategy', tasks, tags=[GraalPythonTags.unittest_arrow]) as task:
1492+
if task:
1493+
run_python_unittests(graalpy_standalone_jvm(), args=["--python.UseNativeStorageStrategy"], report=report(), nonZeroIsFatal=nonZeroIsFatal)
1494+
14901495
with Task('GraalPython HPy tests', tasks, tags=[GraalPythonTags.unittest_hpy]) as task:
14911496
if task:
14921497
run_hpy_unittests(graalpy_standalone_native(), nonZeroIsFatal=nonZeroIsFatal, report=report())

0 commit comments

Comments
 (0)