Skip to content

Commit ca2c035

Browse files
msimacekmsimacek
committed
[GR-40128] Make graalpyspark unittests work under bytecode interpreter
PullRequest: graalpython/2369
2 parents ea1a66b + 47d9e75 commit ca2c035

19 files changed

+277
-76
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
import static com.oracle.graal.python.runtime.exception.PythonErrorType.RuntimeError;
8787
import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError;
8888
import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
89+
import static com.oracle.graal.python.util.PythonUtils.objectArrayToTruffleStringArray;
8990
import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
9091

9192
import java.math.BigInteger;
@@ -2486,15 +2487,16 @@ PCode call(VirtualFrame frame, @SuppressWarnings("unused") Object cls, int argco
24862487
PTuple freevars, PTuple cellvars,
24872488
@CachedLibrary(limit = "1") PythonBufferAccessLibrary bufferLib,
24882489
@Cached CodeNodes.CreateCodeNode createCodeNode,
2489-
@Cached GetObjectArrayNode getObjectArrayNode) {
2490+
@Cached GetObjectArrayNode getObjectArrayNode,
2491+
@Cached CastToTruffleStringNode castToTruffleStringNode) {
24902492
byte[] codeBytes = bufferLib.getCopiedByteArray(codestring);
24912493
byte[] lnotabBytes = bufferLib.getCopiedByteArray(lnotab);
24922494

24932495
Object[] constantsArr = getObjectArrayNode.execute(constants);
2494-
Object[] namesArr = getObjectArrayNode.execute(names);
2495-
Object[] varnamesArr = getObjectArrayNode.execute(varnames);
2496-
Object[] freevarsArr = getObjectArrayNode.execute(freevars);
2497-
Object[] cellcarsArr = getObjectArrayNode.execute(cellvars);
2496+
TruffleString[] namesArr = objectArrayToTruffleStringArray(getObjectArrayNode.execute(names), castToTruffleStringNode);
2497+
TruffleString[] varnamesArr = objectArrayToTruffleStringArray(getObjectArrayNode.execute(varnames), castToTruffleStringNode);
2498+
TruffleString[] freevarsArr = objectArrayToTruffleStringArray(getObjectArrayNode.execute(freevars), castToTruffleStringNode);
2499+
TruffleString[] cellcarsArr = objectArrayToTruffleStringArray(getObjectArrayNode.execute(cellvars), castToTruffleStringNode);
24982500

24992501
return createCodeNode.execute(frame, argcount, posonlyargcount, kwonlyargcount,
25002502
nlocals, stacksize, flags,

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextCodeBuiltins.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
import static com.oracle.graal.python.util.PythonUtils.EMPTY_BYTE_ARRAY;
4444
import static com.oracle.graal.python.util.PythonUtils.EMPTY_OBJECT_ARRAY;
45+
import static com.oracle.graal.python.util.PythonUtils.EMPTY_TRUFFLESTRING_ARRAY;
4546

4647
import java.util.List;
4748

@@ -126,8 +127,8 @@ abstract static class PyCodeNewEmpty extends PythonTernaryBuiltinNode {
126127
static PCode newEmpty(TruffleString filename, TruffleString funcname, int lineno,
127128
@Cached CodeNodes.CreateCodeNode createCodeNode) {
128129
return createCodeNode.execute(null, 0, 0, 0, 0, 0, 0,
129-
EMPTY_BYTE_ARRAY, EMPTY_OBJECT_ARRAY, EMPTY_OBJECT_ARRAY, EMPTY_OBJECT_ARRAY, EMPTY_OBJECT_ARRAY, EMPTY_OBJECT_ARRAY,
130-
filename, funcname, lineno, EMPTY_BYTE_ARRAY);
130+
EMPTY_BYTE_ARRAY, EMPTY_OBJECT_ARRAY, EMPTY_TRUFFLESTRING_ARRAY, EMPTY_TRUFFLESTRING_ARRAY, EMPTY_TRUFFLESTRING_ARRAY,
131+
EMPTY_TRUFFLESTRING_ARRAY, filename, funcname, lineno, EMPTY_BYTE_ARRAY);
131132
}
132133
}
133134
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeBuiltins.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REPR__;
3434
import static com.oracle.graal.python.nodes.StringLiterals.T_NONE;
3535
import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING;
36+
import static com.oracle.graal.python.util.PythonUtils.objectArrayToTruffleStringArray;
3637

3738
import java.util.Arrays;
3839
import java.util.List;
@@ -54,6 +55,7 @@
5455
import com.oracle.graal.python.nodes.function.builtins.PythonClinicBuiltinNode;
5556
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
5657
import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
58+
import com.oracle.graal.python.nodes.util.CastToTruffleStringNode;
5759
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
5860
import com.oracle.graal.python.util.PythonUtils;
5961
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
@@ -409,6 +411,7 @@ PCode create(VirtualFrame frame, PCode self, int coArgcount,
409411
Object[] coCellvars, TruffleString coFilename,
410412
TruffleString coName, Object coLnotab,
411413
@Cached CodeNodes.CreateCodeNode createCodeNode,
414+
@Cached CastToTruffleStringNode castToTruffleStringNode,
412415
@CachedLibrary(limit = "2") PythonBufferAccessLibrary bufferLib) {
413416
try {
414417
return createCodeNode.execute(frame,
@@ -420,10 +423,10 @@ PCode create(VirtualFrame frame, PCode self, int coArgcount,
420423
coFlags == -1 ? self.co_flags() : coFlags,
421424
PGuards.isNone(coCode) ? self.getCodestring() : bufferLib.getInternalOrCopiedByteArray(coCode),
422425
coConsts.length == 0 ? null : coConsts,
423-
coNames.length == 0 ? self.getNames() : coNames,
424-
coVarnames.length == 0 ? self.getVarnames() : coVarnames,
425-
coFreevars.length == 0 ? self.getFreeVars() : coFreevars,
426-
coCellvars.length == 0 ? self.getCellVars() : coCellvars,
426+
coNames.length == 0 ? null : objectArrayToTruffleStringArray(coNames, castToTruffleStringNode),
427+
coVarnames.length == 0 ? null : objectArrayToTruffleStringArray(coVarnames, castToTruffleStringNode),
428+
coFreevars.length == 0 ? null : objectArrayToTruffleStringArray(coFreevars, castToTruffleStringNode),
429+
coCellvars.length == 0 ? null : objectArrayToTruffleStringArray(coCellvars, castToTruffleStringNode),
427430
coFilename.isEmpty() ? self.co_filename() : coFilename,
428431
coName.isEmpty() ? self.co_name() : coName,
429432
coFirstlineno == -1 ? self.co_firstlineno() : coFirstlineno,

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -40,25 +40,20 @@
4040
*/
4141
package com.oracle.graal.python.builtins.objects.code;
4242

43-
import static com.oracle.graal.python.nodes.truffle.TruffleStringMigrationPythonTypes.assertNoJavaString;
44-
45-
import java.util.ArrayList;
46-
import java.util.List;
43+
import java.util.Arrays;
4744

4845
import org.graalvm.polyglot.io.ByteSequence;
4946

5047
import com.oracle.graal.python.PythonLanguage;
5148
import com.oracle.graal.python.builtins.modules.MarshalModuleBuiltins;
5249
import com.oracle.graal.python.builtins.objects.function.Signature;
53-
import com.oracle.graal.python.builtins.objects.str.PString;
5450
import com.oracle.graal.python.compiler.CodeUnit;
5551
import com.oracle.graal.python.nodes.IndirectCallNode;
5652
import com.oracle.graal.python.nodes.PNodeWithContext;
5753
import com.oracle.graal.python.nodes.PRootNode;
5854
import com.oracle.graal.python.nodes.bytecode.PBytecodeGeneratorFunctionRootNode;
5955
import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
6056
import com.oracle.graal.python.nodes.util.BadOPCodeNode;
61-
import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
6257
import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
6358
import com.oracle.graal.python.runtime.PythonContext;
6459
import com.oracle.graal.python.runtime.PythonOptions;
@@ -101,8 +96,8 @@ public Assumption needNotPassExceptionAssumption() {
10196
public PCode execute(VirtualFrame frame, int argcount,
10297
int posonlyargcount, int kwonlyargcount,
10398
int nlocals, int stacksize, int flags,
104-
byte[] codedata, Object[] constants, Object[] names,
105-
Object[] varnames, Object[] freevars, Object[] cellvars,
99+
byte[] codedata, Object[] constants, TruffleString[] names,
100+
TruffleString[] varnames, TruffleString[] freevars, TruffleString[] cellvars,
106101
TruffleString filename, TruffleString name, int firstlineno,
107102
byte[] lnotab) {
108103

@@ -122,8 +117,8 @@ public PCode execute(VirtualFrame frame, int argcount,
122117
private static PCode createCode(PythonLanguage language, PythonContext context, @SuppressWarnings("unused") int argcount,
123118
@SuppressWarnings("unused") int posonlyargcount, @SuppressWarnings("unused") int kwonlyargcount,
124119
int nlocals, int stacksize, int flags,
125-
byte[] codedata, Object[] constants, Object[] names,
126-
Object[] varnames, Object[] freevars, Object[] cellvars,
120+
byte[] codedata, Object[] constants, TruffleString[] names,
121+
TruffleString[] varnames, TruffleString[] freevars, TruffleString[] cellvars,
127122
TruffleString filename, TruffleString name, int firstlineno,
128123
byte[] lnotab) {
129124

@@ -132,7 +127,7 @@ private static PCode createCode(PythonLanguage language, PythonContext context,
132127
ct = language.createCachedCallTarget(l -> new BadOPCodeNode(l, name), BadOPCodeNode.class, filename, name);
133128
} else {
134129
if (context.getOption(PythonOptions.EnableBytecodeInterpreter)) {
135-
ct = create().deserializeForBytecodeInterpreter(language, codedata);
130+
ct = create().deserializeForBytecodeInterpreter(language, codedata, cellvars, freevars);
136131
} else {
137132
RootNode rootNode = context.getSerializer().deserialize(context, codedata, toStringArray(cellvars), toStringArray(freevars));
138133
ct = PythonUtils.getOrCreateCallTarget(rootNode);
@@ -146,8 +141,17 @@ private static PCode createCode(PythonLanguage language, PythonContext context,
146141
firstlineno, lnotab);
147142
}
148143

149-
private RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage language, byte[] data) {
144+
private RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage language, byte[] data, TruffleString[] cellvars, TruffleString[] freevars) {
150145
CodeUnit code = MarshalModuleBuiltins.deserializeCodeUnit(data);
146+
if (cellvars != null && !Arrays.equals(code.cellvars, cellvars) || freevars != null && !Arrays.equals(code.freevars, freevars)) {
147+
code = new CodeUnit(code.name, code.qualname, code.argCount, code.kwOnlyArgCount, code.positionalOnlyArgCount, code.stacksize, code.code,
148+
code.srcOffsetTable, code.flags, code.names, code.varnames,
149+
cellvars != null ? cellvars : code.cellvars, freevars != null ? freevars : code.freevars,
150+
code.cell2arg, code.constants, code.primitiveConstants, code.exceptionHandlerRanges, code.conditionProfileCount,
151+
code.startOffset, code.startLine,
152+
code.outputCanQuicken, code.variableShouldUnbox,
153+
code.generalizeInputsMap, code.generalizeVarsMap);
154+
}
151155
RootNode rootNode = new PBytecodeRootNode(language, code, null, null);
152156
if (code.isGeneratorOrCoroutine()) {
153157
rootNode = new PBytecodeGeneratorFunctionRootNode(language, rootNode.getFrameDescriptor(), (PBytecodeRootNode) rootNode, code.name);
@@ -176,18 +180,15 @@ public static PCode createCode(PythonContext context, int flags, byte[] codedata
176180
}
177181

178182
@TruffleBoundary
179-
private static String[] toStringArray(Object[] array) {
180-
List<String> list = new ArrayList<>(array.length);
181-
for (Object item : array) {
182-
item = assertNoJavaString(item);
183-
if (item instanceof TruffleString) {
184-
list.add(((TruffleString) item).toJavaStringUncached());
185-
}
186-
if (item instanceof PString) {
187-
list.add(CastToJavaStringNode.getUncached().execute(item));
188-
}
183+
private static String[] toStringArray(TruffleString[] array) {
184+
if (array == null) {
185+
return null;
186+
}
187+
String[] result = new String[array.length];
188+
for (int i = 0; i < array.length; i++) {
189+
result[i] = array[i].toJavaStringUncached();
189190
}
190-
return list.toArray(new String[list.size()]);
191+
return result;
191192
}
192193

193194
public static CreateCodeNode create() {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyObjectCallMethodObjArgs.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@
4242

4343
import com.oracle.graal.python.builtins.objects.function.PKeyword;
4444
import com.oracle.graal.python.builtins.objects.type.SpecialMethodSlot;
45+
import com.oracle.graal.python.nodes.call.BoundDescriptor;
4546
import com.oracle.graal.python.nodes.call.CallNode;
4647
import com.oracle.graal.python.nodes.call.special.CallBinaryMethodNode;
4748
import com.oracle.graal.python.nodes.call.special.CallQuaternaryMethodNode;
4849
import com.oracle.graal.python.nodes.call.special.CallTernaryMethodNode;
4950
import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
50-
import com.oracle.graal.python.nodes.call.special.MaybeBindDescriptorNode.BoundDescriptor;
5151
import com.oracle.graal.python.util.PythonUtils;
5252
import com.oracle.truffle.api.dsl.Cached;
5353
import com.oracle.truffle.api.dsl.Cached.Shared;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyObjectGetMethod.java

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,15 @@
5353
import com.oracle.graal.python.nodes.attributes.LookupAttributeInMRONode;
5454
import com.oracle.graal.python.nodes.attributes.LookupCallableSlotInMRONode;
5555
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode;
56+
import com.oracle.graal.python.nodes.call.BoundDescriptor;
57+
import com.oracle.graal.python.nodes.call.CallNode;
58+
import com.oracle.graal.python.nodes.call.ForeignMethod;
5659
import com.oracle.graal.python.nodes.call.special.CallTernaryMethodNode;
60+
import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
5761
import com.oracle.graal.python.nodes.call.special.MaybeBindDescriptorNode;
58-
import com.oracle.graal.python.nodes.call.special.MaybeBindDescriptorNode.BoundDescriptor;
5962
import com.oracle.graal.python.nodes.object.GetClassNode;
63+
import com.oracle.graal.python.nodes.object.IsForeignObjectNode;
64+
import com.oracle.graal.python.runtime.GilNode;
6065
import com.oracle.truffle.api.dsl.Bind;
6166
import com.oracle.truffle.api.dsl.Cached;
6267
import com.oracle.truffle.api.dsl.Cached.Shared;
@@ -65,6 +70,8 @@
6570
import com.oracle.truffle.api.dsl.Specialization;
6671
import com.oracle.truffle.api.frame.Frame;
6772
import com.oracle.truffle.api.frame.VirtualFrame;
73+
import com.oracle.truffle.api.interop.InteropLibrary;
74+
import com.oracle.truffle.api.library.CachedLibrary;
6875
import com.oracle.truffle.api.nodes.Node;
6976
import com.oracle.truffle.api.profiles.BranchProfile;
7077
import com.oracle.truffle.api.strings.TruffleString;
@@ -73,8 +80,9 @@
7380
* Equivalent of _PyObject_GetMethod. Like CPython, the node uses {@link PyObjectGetAttr} for any
7481
* object that does not have the generic {@code object.__getattribute__}. For the generic {@code
7582
* object.__getattribute__} the node inlines the default logic but without binding methods, and
76-
* falls back to looking into the object dict. Returns something that can be handled by our
77-
* CallSpecial nodes.
83+
* falls back to looking into the object dict. Returns something that can be handled by
84+
* {@link CallNode} or one of the {@code CallNAryMethodNode} nodes, like
85+
* {@link CallUnaryMethodNode}.
7886
*/
7987
@GenerateUncached
8088
@ImportStatic(SpecialMethodSlot.class)
@@ -97,16 +105,18 @@ protected static boolean isObjectGetAttribute(Object lazyClass) {
97105
return getattributeSlot == BuiltinMethodDescriptors.OBJ_GET_ATTRIBUTE && getattrSlot == PNone.NO_VALUE;
98106
}
99107

100-
@Specialization(guards = "!isObjectGetAttribute(lazyClass)", limit = "1")
108+
@Specialization(guards = {"!isForeignObjectNode.execute(receiver)", "!isObjectGetAttribute(lazyClass)"}, limit = "1")
101109
static Object getGenericAttr(Frame frame, Object receiver, TruffleString name,
110+
@SuppressWarnings("unused") @Shared("isForeign") @Cached IsForeignObjectNode isForeignObjectNode,
102111
@SuppressWarnings("unused") @Shared("getClassNode") @Cached GetClassNode getClass,
103112
@SuppressWarnings("unused") @Bind("getClass.execute(receiver)") Object lazyClass,
104113
@Cached PyObjectGetAttr getAttr) {
105114
return new BoundDescriptor(getAttr.execute(frame, receiver, name));
106115
}
107116

108-
@Specialization(guards = {"isObjectGetAttribute(lazyClass)", "name == cachedName"}, limit = "1")
117+
@Specialization(guards = {"!isForeignObjectNode.execute(receiver)", "isObjectGetAttribute(lazyClass)", "name == cachedName"}, limit = "1")
109118
static Object getFixedAttr(VirtualFrame frame, Object receiver, @SuppressWarnings("unused") TruffleString name,
119+
@SuppressWarnings("unused") @Shared("isForeign") @Cached IsForeignObjectNode isForeignObjectNode,
110120
@SuppressWarnings("unused") @Shared("getClassNode") @Cached GetClassNode getClass,
111121
@Bind("getClass.execute(receiver)") Object lazyClass,
112122
@SuppressWarnings("unused") @Cached("name") TruffleString cachedName,
@@ -163,8 +173,9 @@ static Object getFixedAttr(VirtualFrame frame, Object receiver, @SuppressWarning
163173
}
164174

165175
// No explicit branch profiling when we're looking up multiple things
166-
@Specialization(guards = "isObjectGetAttribute(lazyClass)", replaces = "getFixedAttr", limit = "1")
176+
@Specialization(guards = {"!isForeignObjectNode.execute(receiver)", "isObjectGetAttribute(lazyClass)"}, replaces = "getFixedAttr", limit = "1")
167177
static Object getDynamicAttr(Frame frame, Object receiver, TruffleString name,
178+
@SuppressWarnings("unused") @Shared("isForeign") @Cached IsForeignObjectNode isForeignObjectNode,
168179
@SuppressWarnings("unused") @Shared("getClassNode") @Cached GetClassNode getClass,
169180
@Bind("getClass.execute(receiver)") Object lazyClass,
170181
@Cached LookupAttributeInMRONode.Dynamic lookupNode,
@@ -205,4 +216,26 @@ static Object getDynamicAttr(Frame frame, Object receiver, TruffleString name,
205216
}
206217
throw raiseNode.raise(AttributeError, ErrorMessages.OBJ_P_HAS_NO_ATTR_S, receiver, name);
207218
}
219+
220+
@Specialization(guards = "isForeignObjectNode.execute(receiver)", limit = "3")
221+
Object getForeignMethod(VirtualFrame frame, Object receiver, TruffleString name,
222+
@SuppressWarnings("unused") @Cached IsForeignObjectNode isForeignObjectNode,
223+
@Cached TruffleString.ToJavaStringNode toJavaString,
224+
@CachedLibrary("receiver") InteropLibrary lib,
225+
@Cached PyObjectGetAttr getAttr,
226+
@Cached GilNode gil) {
227+
String jName = toJavaString.execute(name);
228+
boolean memberInvocable;
229+
gil.release(true);
230+
try {
231+
memberInvocable = lib.isMemberInvocable(receiver, jName);
232+
} finally {
233+
gil.acquire();
234+
}
235+
if (memberInvocable) {
236+
return new BoundDescriptor(new ForeignMethod(receiver, jName));
237+
} else {
238+
return new BoundDescriptor(getAttr.execute(frame, receiver, name));
239+
}
240+
}
208241
}

0 commit comments

Comments
 (0)