Skip to content

Commit b1d349f

Browse files
timfeltimfel
committed
[GR-23280] Pass a few more builtin tests
PullRequest: graalpython/1090
2 parents ab8c9de + 6576f40 commit b1d349f

File tree

18 files changed

+246
-95
lines changed

18 files changed

+246
-95
lines changed

‎docs/contributor/CONTRIBUTING.md

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -167,16 +167,11 @@ implementation isn't quite ready, so you have to put the flags somewhere else to
167167
debug. You can see `-debug-java` and `--inspect` below, to debug in Java
168168
debugger or Chromium, respectively.
169169

170-
ENABLE_CPYTHON_TAGGED_UNITTESTS=true mx python3 \
171-
graalpython/com.oracle.graal.python.test/src/graalpytest.py \
172-
[-debug-java] [--inspect] \
173-
graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py \
174-
-k NAME-OF-CPYTHON-UNITTEST
170+
mx python-run-cpython-unittest [-debug-java] [--inspect] NAME-OF-CPYTHON-UNITTEST
175171

176172
A tag file can be regenerated with
177173

178-
mx python graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py \
179-
--retag NAME-OF-CPYTHON-UNITTEST
174+
mx python-retag-unittests NAME-OF-CPYTHON-UNITTEST
180175

181176
There's also multiple other gates that may fail with changes. One of these is
182177
our *style* gate, which checks formatting rules and copyrights. To auto-fix most

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_builtin.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,16 @@
66
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_bytearray_translate
77
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_callable
88
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_cmp
9+
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_compile_async_generator
10+
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_compile_top_level_await
11+
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_construct_singletons
912
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_delattr
13+
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_eval
1014
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_exec
1115
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_exec_redirected
1216
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_filter
1317
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_filter_pickle
18+
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_general_eval
1419
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_hex
1520
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_id
1621
*graalpython.lib-python.3.test.test_builtin.BuiltinTest.test_isinstance
@@ -45,3 +50,5 @@
4550
*graalpython.lib-python.3.test.test_builtin.TestBreakpoint.test_envar_unimportable
4651
*graalpython.lib-python.3.test.test_builtin.TestSorted.test_basic
4752
*graalpython.lib-python.3.test.test_builtin.TestSorted.test_inputtypes
53+
*graalpython.lib-python.3.test.test_builtin.TestType.test_type_doc
54+
*graalpython.lib-python.3.test.test_builtin.TestType.test_type_nokwargs

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

Lines changed: 49 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
8585

8686
import java.math.BigInteger;
87+
import java.nio.charset.StandardCharsets;
8788
import java.util.Arrays;
8889
import java.util.List;
8990
import java.util.Locale;
@@ -160,6 +161,7 @@
160161
import com.oracle.graal.python.nodes.BuiltinNames;
161162
import com.oracle.graal.python.nodes.ErrorMessages;
162163
import com.oracle.graal.python.nodes.PGuards;
164+
import com.oracle.graal.python.nodes.SpecialAttributeNames;
163165
import com.oracle.graal.python.nodes.SpecialMethodNames;
164166
import com.oracle.graal.python.nodes.attributes.GetAttributeNode;
165167
import com.oracle.graal.python.nodes.attributes.GetAttributeNode.GetAnyAttributeNode;
@@ -2146,23 +2148,24 @@ public abstract static class TypeNode extends PythonBuiltinNode {
21462148
@Specialization(guards = {"isNoValue(bases)", "isNoValue(dict)"})
21472149
@SuppressWarnings("unused")
21482150
Object type(Object cls, Object obj, PNone bases, PNone dict, PKeyword[] kwds,
2149-
@Cached("create()") GetClassNode getClass) {
2151+
@Cached GetClassNode getClass) {
21502152
return getClass.execute(obj);
21512153
}
21522154

2153-
@Specialization
2154-
Object type(VirtualFrame frame, Object cls, String name, PTuple bases, PDict namespace, PKeyword[] kwds,
2155+
@Specialization(guards = "isString(wName)")
2156+
Object typeNew(VirtualFrame frame, Object cls, Object wName, PTuple bases, PDict namespace, PKeyword[] kwds,
21552157
@CachedLibrary(limit = "4") PythonObjectLibrary lib,
2156-
@CachedLibrary(limit = "1") HashingStorageLibrary nslib,
2157-
@CachedLibrary(limit = "1") HashingStorageLibrary glib,
2158+
@CachedLibrary(limit = "2") HashingStorageLibrary nslib,
21582159
@Cached BranchProfile updatedStorage,
21592160
@Cached("create(__NEW__)") LookupInheritedAttributeNode getNewFuncNode,
21602161
@Cached("create(__INIT_SUBCLASS__)") GetAttributeNode getInitSubclassNode,
21612162
@Cached("create(__SET_NAME__)") LookupInheritedAttributeNode getSetNameNode,
2163+
@Cached CastToJavaStringNode castStr,
21622164
@Cached CallNode callSetNameNode,
21632165
@Cached CallNode callInitSubclassNode,
21642166
@Cached CallNode callNewFuncNode) {
21652167
// Determine the proper metatype to deal with this
2168+
String name = castStr.execute(wName);
21662169
Object metaclass = calculate_metaclass(frame, cls, bases, lib);
21672170
if (metaclass != cls) {
21682171
Object newFunc = getNewFuncNode.execute(metaclass);
@@ -2195,7 +2198,7 @@ Object type(VirtualFrame frame, Object cls, String name, PTuple bases, PDict nam
21952198
PFrame callerFrame = getReadCallerFrameNode().executeWith(frame, 0);
21962199
PythonObject globals = callerFrame.getGlobals();
21972200
if (globals != null) {
2198-
String moduleName = getModuleNameFromGlobals(globals, glib);
2201+
String moduleName = getModuleNameFromGlobals(globals, nslib);
21992202
if (moduleName != null) {
22002203
ensureWriteAttrNode().execute(frame, newType, __MODULE__, moduleName);
22012204
}
@@ -2225,6 +2228,19 @@ Object type(VirtualFrame frame, Object cls, String name, PTuple bases, PDict nam
22252228
}
22262229
}
22272230

2231+
@Fallback
2232+
Object generic(@SuppressWarnings("unused") Object cls, @SuppressWarnings("unused") Object name, Object bases, Object namespace, @SuppressWarnings("unused") Object kwds) {
2233+
if (!(bases instanceof PTuple)) {
2234+
throw raise(TypeError, ErrorMessages.ARG_D_MUST_BE_S_NOT_P, "type.__new__()", 2, "tuple", bases);
2235+
} else if (namespace == PNone.NO_VALUE) {
2236+
throw raise(TypeError, ErrorMessages.TAKES_D_OR_D_ARGS, "type()", 1, 3);
2237+
} else if (!(namespace instanceof PDict)) {
2238+
throw raise(TypeError, ErrorMessages.ARG_D_MUST_BE_S_NOT_P, "type.__new__()", 3, "dict", bases);
2239+
} else {
2240+
throw CompilerDirectives.shouldNotReachHere("type fallback reached incorrectly");
2241+
}
2242+
}
2243+
22282244
private String getModuleNameFromGlobals(PythonObject globals, HashingStorageLibrary hlib) {
22292245
Object nameAttr;
22302246
if (globals instanceof PythonModule) {
@@ -2295,6 +2311,22 @@ private PythonClass typeMetaclass(VirtualFrame frame, String name, PTuple bases,
22952311
} else {
22962312
pythonClass.setAttribute(key, value);
22972313
}
2314+
} else if (SpecialAttributeNames.__DOC__.equals(key)) {
2315+
// CPython sets tp_doc to a copy of dict['__doc__'], if that is a string. It
2316+
// forcibly encodes the string as UTF-8, and raises an error if that is not
2317+
// possible.
2318+
String doc = null;
2319+
if (value instanceof String) {
2320+
doc = (String) value;
2321+
} else if (value instanceof PString) {
2322+
doc = ((PString) value).getValue();
2323+
}
2324+
if (doc != null) {
2325+
if (!canEncode(doc)) {
2326+
throw raise(PythonBuiltinClassType.UnicodeEncodeError, ErrorMessages.CANNOT_ENCODE_DOCSTR, doc);
2327+
}
2328+
}
2329+
pythonClass.setAttribute(key, value);
22982330
} else {
22992331
pythonClass.setAttribute(key, value);
23002332
}
@@ -2365,6 +2397,11 @@ private PythonClass typeMetaclass(VirtualFrame frame, String name, PTuple bases,
23652397
return pythonClass;
23662398
}
23672399

2400+
@TruffleBoundary
2401+
private static boolean canEncode(String doc) {
2402+
return StandardCharsets.UTF_8.newEncoder().canEncode(doc);
2403+
}
2404+
23682405
@TruffleBoundary
23692406
private PTuple copySlots(String className, SequenceStorage slotList, int slotlen, boolean add_dict, boolean add_weak, PDict namespace, HashingStorageLibrary nslib) {
23702407
SequenceStorage newSlots = new ObjectSequenceStorage(slotlen - PInt.intValue(add_dict) - PInt.intValue(add_weak));
@@ -2659,44 +2696,30 @@ Object doNative(PythonAbstractNativeObject self, Object[] varargs, PKeyword[] kw
26592696
@Builtin(name = "NotImplementedType", minNumOfPositionalArgs = 1, constructsClass = PythonBuiltinClassType.PNotImplemented, isPublic = false)
26602697
@GenerateNodeFactory
26612698
public abstract static class NotImplementedTypeNode extends PythonBuiltinNode {
2662-
protected PythonBuiltinClass getNotImplementedClass() {
2663-
return getCore().lookupType(PythonBuiltinClassType.PNotImplemented);
2664-
}
2665-
2699+
@SuppressWarnings("unused")
26662700
@Specialization
26672701
public PNotImplemented module(Object cls) {
2668-
if (cls != getNotImplementedClass()) {
2669-
throw raise(TypeError, ErrorMessages.OBJ_ISNT_CALLABLE, "NotImplementedType");
2670-
} else {
2671-
return PNotImplemented.NOT_IMPLEMENTED;
2672-
}
2702+
return PNotImplemented.NOT_IMPLEMENTED;
26732703
}
26742704
}
26752705

2676-
@Builtin(name = "ellipsis", takesVarArgs = true, takesVarKeywordArgs = true, constructsClass = PythonBuiltinClassType.PEllipsis, isPublic = false)
2706+
@Builtin(name = "ellipsis", minNumOfPositionalArgs = 1, constructsClass = PythonBuiltinClassType.PEllipsis, isPublic = false)
26772707
@GenerateNodeFactory
26782708
public abstract static class EllipsisTypeNode extends PythonBuiltinNode {
26792709
@SuppressWarnings("unused")
26802710
@Specialization
2681-
public PEllipsis call(Object cls, Object args, Object kwds) {
2711+
public PEllipsis call(Object cls) {
26822712
return PEllipsis.INSTANCE;
26832713
}
26842714
}
26852715

26862716
@Builtin(name = "NoneType", minNumOfPositionalArgs = 1, constructsClass = PythonBuiltinClassType.PNone, isPublic = false)
26872717
@GenerateNodeFactory
26882718
public abstract static class NoneTypeNode extends PythonBuiltinNode {
2689-
protected PythonBuiltinClass getNoneClass() {
2690-
return getCore().lookupType(PythonBuiltinClassType.PNone);
2691-
}
2692-
2719+
@SuppressWarnings("unused")
26932720
@Specialization
26942721
public PNone module(Object cls) {
2695-
if (cls != getNoneClass()) {
2696-
throw raise(TypeError, ErrorMessages.IS_NOT_SUBTYPE_OF, "NoneType.__new__", cls, "NoneType");
2697-
} else {
2698-
return PNone.NONE;
2699-
}
2722+
return PNone.NONE;
27002723
}
27012724
}
27022725

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 54 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,6 @@
8282
import com.oracle.graal.python.builtins.modules.BuiltinFunctionsFactory.GetAttrNodeFactory;
8383
import com.oracle.graal.python.builtins.modules.BuiltinFunctionsFactory.GlobalsNodeFactory;
8484
import com.oracle.graal.python.builtins.objects.PNone;
85-
import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
8685
import com.oracle.graal.python.builtins.objects.bytes.BytesUtils;
8786
import com.oracle.graal.python.builtins.objects.bytes.PBytes;
8887
import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
@@ -183,6 +182,7 @@
183182
import com.oracle.truffle.api.dsl.TypeSystemReference;
184183
import com.oracle.truffle.api.frame.Frame;
185184
import com.oracle.truffle.api.frame.VirtualFrame;
185+
import com.oracle.truffle.api.interop.InteropLibrary;
186186
import com.oracle.truffle.api.interop.UnsupportedMessageException;
187187
import com.oracle.truffle.api.library.CachedLibrary;
188188
import com.oracle.truffle.api.nodes.ExplodeLoop;
@@ -747,26 +747,6 @@ public CompileNode() {
747747

748748
public abstract PCode execute(VirtualFrame frame, Object source, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize);
749749

750-
@Specialization
751-
PCode compile(VirtualFrame frame, PBytes pBytes, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize,
752-
@Cached("create()") BytesNodes.ToBytesNode toBytesNode) {
753-
try {
754-
byte[] bytes = toBytesNode.execute(frame, pBytes);
755-
Charset charset = PythonFileDetector.findEncodingStrict(bytes);
756-
return compile(createString(bytes, charset), filename, mode, kwFlags, kwDontInherit, kwOptimize);
757-
} catch (PythonFileDetector.InvalidEncodingException e) {
758-
throw handleInvalidEncoding(filename, e);
759-
}
760-
}
761-
762-
@TruffleBoundary
763-
private RuntimeException handleInvalidEncoding(String filename, PythonFileDetector.InvalidEncodingException e) {
764-
PythonContext context = getContext();
765-
// Create non-empty source to avoid overwriting the message with "unexpected EOF"
766-
Source source = PythonLanguage.newSource(context, " ", filename, mayBeFromFile);
767-
throw getCore().raiseInvalidSyntax(source, source.createUnavailableSection(), "encoding problem: %s", e.getEncodingName());
768-
}
769-
770750
@SuppressWarnings("unused")
771751
@Specialization
772752
@TruffleBoundary
@@ -805,10 +785,59 @@ PCode compile(String expression, String filename, String mode, Object kwFlags, O
805785
return factory().createCode(ct);
806786
}
807787

808-
@SuppressWarnings("unused")
809-
@Specialization
810-
PCode compile(PCode code, String filename, String mode, Object flags, Object dontInherit, Object optimize) {
811-
return code;
788+
@Specialization(limit = "3")
789+
PCode generic(VirtualFrame frame, Object wSource, Object wFilename, Object wMode, Object kwFlags, Object kwDontInherit, Object kwOptimize,
790+
@Cached CastToJavaStringNode castStr,
791+
@CachedLibrary("wSource") InteropLibrary interopLib,
792+
@CachedLibrary(limit = "3") PythonObjectLibrary lib) {
793+
if (wSource instanceof PCode) {
794+
return (PCode) wSource;
795+
}
796+
String filename = lib.asPathWithState(wFilename, PArguments.getThreadState(frame));
797+
String mode;
798+
try {
799+
mode = castStr.execute(wMode);
800+
} catch (CannotCastException e) {
801+
throw raise(TypeError, ErrorMessages.ARG_S_MUST_BE_S_NOT_P, "compile()", "mode", "str", wMode);
802+
}
803+
String source = sourceAsString(wSource, filename, interopLib, lib);
804+
return compile(source, filename, mode, kwFlags, kwDontInherit, kwOptimize);
805+
}
806+
807+
// modeled after _Py_SourceAsString
808+
String sourceAsString(Object source, String filename, InteropLibrary interopLib, PythonObjectLibrary pyLib) {
809+
if (interopLib.isString(source)) {
810+
try {
811+
return interopLib.asString(source);
812+
} catch (UnsupportedMessageException e) {
813+
throw CompilerDirectives.shouldNotReachHere(e);
814+
}
815+
} else if (pyLib.isBuffer(source)) {
816+
// cpython checks for bytes and bytearray separately, but we deal with it as
817+
// buffers, since that's fast for us anyway
818+
try {
819+
byte[] bytes;
820+
try {
821+
bytes = pyLib.getBufferBytes(source);
822+
} catch (UnsupportedMessageException e) {
823+
throw CompilerDirectives.shouldNotReachHere(e);
824+
}
825+
Charset charset = PythonFileDetector.findEncodingStrict(bytes);
826+
return createString(bytes, charset);
827+
} catch (PythonFileDetector.InvalidEncodingException e) {
828+
throw handleInvalidEncoding(filename, e);
829+
}
830+
} else {
831+
throw raise(TypeError, ErrorMessages.ARG_D_MUST_BE_S, "compile()", 1, "string, bytes or AST object");
832+
}
833+
}
834+
835+
@TruffleBoundary
836+
private RuntimeException handleInvalidEncoding(String filename, PythonFileDetector.InvalidEncodingException e) {
837+
PythonContext context = getContext();
838+
// Create non-empty source to avoid overwriting the message with "unexpected EOF"
839+
Source source = PythonLanguage.newSource(context, " ", filename, mayBeFromFile);
840+
throw getCore().raiseInvalidSyntax(source, source.createUnavailableSection(), "encoding problem: %s", e.getEncodingName());
812841
}
813842

814843
@TruffleBoundary

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/PBytes.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
3232
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
33+
import com.oracle.graal.python.builtins.objects.function.PArguments.ThreadState;
3334
import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
3435
import com.oracle.graal.python.nodes.PRaiseNode;
3536
import com.oracle.graal.python.runtime.sequence.PImmutableSequence;
@@ -135,7 +136,8 @@ byte[] getBufferBytes(
135136
}
136137

137138
@ExportMessage
138-
public String asPath(@Cached PRaiseNode raise,
139+
public String asPathWithState(@SuppressWarnings("unused") ThreadState state,
140+
@Cached PRaiseNode raise,
139141
@Shared("toByteArrayNode") @Cached SequenceStorageNodes.ToByteArrayNode toBytes) {
140142
return newString(raise, toBytes.execute(getSequenceStorage()));
141143
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/PString.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,8 @@ static int subclassedString(PString self, ThreadState state,
179179
}
180180

181181
@ExportMessage
182-
public String asPath(@Cached CastToJavaStringNode castToJavaStringNode) {
182+
public String asPathWithState(@SuppressWarnings("unused") ThreadState state,
183+
@Cached CastToJavaStringNode castToJavaStringNode) {
183184
try {
184185
return castToJavaStringNode.execute(this);
185186
} catch (CannotCastException e) {

0 commit comments

Comments
 (0)