Skip to content

Commit b64c096

Browse files
otethalotethal
committed
[GR-23306] Make test_json pass
PullRequest: graalpython/1305
2 parents fb0fb9d + 7cd17b4 commit b64c096

File tree

7 files changed

+401
-23
lines changed

7 files changed

+401
-23
lines changed

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727

2828
import java.io.EOFException;
2929
import java.io.File;
30+
import java.io.FileDescriptor;
31+
import java.io.FileOutputStream;
3032
import java.io.IOException;
3133
import java.io.InputStream;
3234
import java.io.OutputStream;
@@ -385,6 +387,9 @@ private String getExecutable() {
385387

386388
@Override
387389
protected void launch(Builder contextBuilder) {
390+
// prevent the use of System.out/err - they are PrintStreams which suppresses exceptions
391+
contextBuilder.out(new FileOutputStream(FileDescriptor.out));
392+
contextBuilder.err(new FileOutputStream(FileDescriptor.err));
388393
if (!ignoreEnv) {
389394
String pythonpath = System.getenv("PYTHONPATH");
390395
if (pythonpath != null) {

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_json.txt

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
*graalpython.lib-python.3.test.test_json.__init__.TestCTest.test_cjson
2+
*graalpython.lib-python.3.test.test_json.__init__.TestPyTest.test_pyjson
3+
*test.test_json.TestCTest.test_cjson
4+
*test.test_json.TestPyTest.test_pyjson
5+
*test.test_json.test_decode.TestCDecode.test_decimal
6+
*test.test_json.test_decode.TestCDecode.test_decoder_optimizations
7+
*test.test_json.test_decode.TestCDecode.test_deprecated_encode
8+
*test.test_json.test_decode.TestCDecode.test_empty_objects
9+
*test.test_json.test_decode.TestCDecode.test_extra_data
10+
*test.test_json.test_decode.TestCDecode.test_float
11+
*test.test_json.test_decode.TestCDecode.test_invalid_escape
12+
*test.test_json.test_decode.TestCDecode.test_invalid_input_type
13+
*test.test_json.test_decode.TestCDecode.test_keys_reuse
14+
*test.test_json.test_decode.TestCDecode.test_negative_index
15+
*test.test_json.test_decode.TestCDecode.test_object_pairs_hook
16+
*test.test_json.test_decode.TestCDecode.test_string_with_utf8_bom
17+
*test.test_json.test_decode.TestPyDecode.test_decimal
18+
*test.test_json.test_decode.TestPyDecode.test_decoder_optimizations
19+
*test.test_json.test_decode.TestPyDecode.test_deprecated_encode
20+
*test.test_json.test_decode.TestPyDecode.test_empty_objects
21+
*test.test_json.test_decode.TestPyDecode.test_extra_data
22+
*test.test_json.test_decode.TestPyDecode.test_float
23+
*test.test_json.test_decode.TestPyDecode.test_invalid_escape
24+
*test.test_json.test_decode.TestPyDecode.test_invalid_input_type
25+
*test.test_json.test_decode.TestPyDecode.test_keys_reuse
26+
*test.test_json.test_decode.TestPyDecode.test_negative_index
27+
*test.test_json.test_decode.TestPyDecode.test_object_pairs_hook
28+
*test.test_json.test_decode.TestPyDecode.test_string_with_utf8_bom
29+
*test.test_json.test_default.TestCDefault.test_default
30+
*test.test_json.test_default.TestPyDefault.test_default
31+
*test.test_json.test_dump.TestCDump.test_dump
32+
*test.test_json.test_dump.TestCDump.test_dump_skipkeys
33+
*test.test_json.test_dump.TestCDump.test_dumps
34+
*test.test_json.test_dump.TestCDump.test_encode_evil_dict
35+
*test.test_json.test_dump.TestCDump.test_encode_mutated
36+
*test.test_json.test_dump.TestCDump.test_encode_truefalse
37+
*test.test_json.test_dump.TestCDump.test_large_list
38+
*test.test_json.test_dump.TestPyDump.test_dump
39+
*test.test_json.test_dump.TestPyDump.test_dump_skipkeys
40+
*test.test_json.test_dump.TestPyDump.test_dumps
41+
*test.test_json.test_dump.TestPyDump.test_encode_evil_dict
42+
*test.test_json.test_dump.TestPyDump.test_encode_mutated
43+
*test.test_json.test_dump.TestPyDump.test_encode_truefalse
44+
*test.test_json.test_encode_basestring_ascii.TestCEncodeBasestringAscii.test_encode_basestring_ascii
45+
*test.test_json.test_encode_basestring_ascii.TestCEncodeBasestringAscii.test_ordered_dict
46+
*test.test_json.test_encode_basestring_ascii.TestCEncodeBasestringAscii.test_overflow
47+
*test.test_json.test_encode_basestring_ascii.TestCEncodeBasestringAscii.test_sorted_dict
48+
*test.test_json.test_encode_basestring_ascii.TestPyEncodeBasestringAscii.test_encode_basestring_ascii
49+
*test.test_json.test_encode_basestring_ascii.TestPyEncodeBasestringAscii.test_ordered_dict
50+
*test.test_json.test_encode_basestring_ascii.TestPyEncodeBasestringAscii.test_sorted_dict
51+
*test.test_json.test_enum.TestCEnum.test_dict_keys
52+
*test.test_json.test_enum.TestCEnum.test_dict_values
53+
*test.test_json.test_enum.TestCEnum.test_floats
54+
*test.test_json.test_enum.TestCEnum.test_ints
55+
*test.test_json.test_enum.TestCEnum.test_list
56+
*test.test_json.test_enum.TestCEnum.test_weird_floats
57+
*test.test_json.test_enum.TestPyEnum.test_dict_keys
58+
*test.test_json.test_enum.TestPyEnum.test_dict_values
59+
*test.test_json.test_enum.TestPyEnum.test_floats
60+
*test.test_json.test_enum.TestPyEnum.test_ints
61+
*test.test_json.test_enum.TestPyEnum.test_list
62+
*test.test_json.test_enum.TestPyEnum.test_weird_floats
63+
*test.test_json.test_fail.TestCFail.test_extra_data
64+
*test.test_json.test_fail.TestCFail.test_failures
65+
*test.test_json.test_fail.TestCFail.test_linecol
66+
*test.test_json.test_fail.TestCFail.test_non_string_keys_dict
67+
*test.test_json.test_fail.TestCFail.test_not_serializable
68+
*test.test_json.test_fail.TestCFail.test_truncated_input
69+
*test.test_json.test_fail.TestCFail.test_unexpected_data
70+
*test.test_json.test_fail.TestPyFail.test_extra_data
71+
*test.test_json.test_fail.TestPyFail.test_failures
72+
*test.test_json.test_fail.TestPyFail.test_linecol
73+
*test.test_json.test_fail.TestPyFail.test_non_string_keys_dict
74+
*test.test_json.test_fail.TestPyFail.test_not_serializable
75+
*test.test_json.test_fail.TestPyFail.test_truncated_input
76+
*test.test_json.test_fail.TestPyFail.test_unexpected_data
77+
*test.test_json.test_float.TestCFloat.test_allow_nan
78+
*test.test_json.test_float.TestCFloat.test_floats
79+
*test.test_json.test_float.TestCFloat.test_ints
80+
*test.test_json.test_float.TestCFloat.test_out_of_range
81+
*test.test_json.test_float.TestPyFloat.test_allow_nan
82+
*test.test_json.test_float.TestPyFloat.test_floats
83+
*test.test_json.test_float.TestPyFloat.test_ints
84+
*test.test_json.test_float.TestPyFloat.test_out_of_range
85+
*test.test_json.test_indent.TestCIndent.test_indent
86+
*test.test_json.test_indent.TestCIndent.test_indent0
87+
*test.test_json.test_indent.TestPyIndent.test_indent
88+
*test.test_json.test_indent.TestPyIndent.test_indent0
89+
*test.test_json.test_pass1.TestCPass1.test_parse
90+
*test.test_json.test_pass1.TestPyPass1.test_parse
91+
*test.test_json.test_pass2.TestCPass2.test_parse
92+
*test.test_json.test_pass2.TestPyPass2.test_parse
93+
*test.test_json.test_pass3.TestCPass3.test_parse
94+
*test.test_json.test_pass3.TestPyPass3.test_parse
95+
*test.test_json.test_recursion.TestCRecursion.test_defaultrecursion
96+
*test.test_json.test_recursion.TestCRecursion.test_dictrecursion
97+
*test.test_json.test_recursion.TestCRecursion.test_endless_recursion
98+
*test.test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding
99+
*test.test_json.test_recursion.TestCRecursion.test_highly_nested_objects_encoding
100+
*test.test_json.test_recursion.TestCRecursion.test_listrecursion
101+
*test.test_json.test_recursion.TestPyRecursion.test_defaultrecursion
102+
*test.test_json.test_recursion.TestPyRecursion.test_dictrecursion
103+
*test.test_json.test_recursion.TestPyRecursion.test_endless_recursion
104+
*test.test_json.test_recursion.TestPyRecursion.test_highly_nested_objects_decoding
105+
*test.test_json.test_recursion.TestPyRecursion.test_highly_nested_objects_encoding
106+
*test.test_json.test_recursion.TestPyRecursion.test_listrecursion
107+
*test.test_json.test_scanstring.TestCScanstring.test_bad_escapes
108+
*test.test_json.test_scanstring.TestCScanstring.test_overflow
109+
*test.test_json.test_scanstring.TestCScanstring.test_scanstring
110+
*test.test_json.test_scanstring.TestCScanstring.test_surrogates
111+
*test.test_json.test_scanstring.TestPyScanstring.test_bad_escapes
112+
*test.test_json.test_scanstring.TestPyScanstring.test_overflow
113+
*test.test_json.test_scanstring.TestPyScanstring.test_scanstring
114+
*test.test_json.test_scanstring.TestPyScanstring.test_surrogates
115+
*test.test_json.test_separators.TestCSeparators.test_illegal_separators
116+
*test.test_json.test_separators.TestCSeparators.test_separators
117+
*test.test_json.test_separators.TestPySeparators.test_illegal_separators
118+
*test.test_json.test_separators.TestPySeparators.test_separators
119+
*test.test_json.test_speedups.TestDecode.test_bad_bool_args
120+
*test.test_json.test_speedups.TestDecode.test_make_scanner
121+
*test.test_json.test_speedups.TestEncode.test_bad_bool_args
122+
*test.test_json.test_speedups.TestEncode.test_bad_str_encoder
123+
*test.test_json.test_speedups.TestEncode.test_make_encoder
124+
*test.test_json.test_speedups.TestEncode.test_unsortable_keys
125+
*test.test_json.test_speedups.TestSpeedups.test_encode_basestring_ascii
126+
*test.test_json.test_speedups.TestSpeedups.test_scanstring
127+
*test.test_json.test_tool.TestTool.test_broken_pipe_error
128+
*test.test_json.test_tool.TestTool.test_help_flag
129+
*test.test_json.test_tool.TestTool.test_infile_outfile
130+
*test.test_json.test_tool.TestTool.test_infile_stdout
131+
*test.test_json.test_tool.TestTool.test_jsonlines
132+
*test.test_json.test_tool.TestTool.test_non_ascii_infile
133+
*test.test_json.test_tool.TestTool.test_sort_keys_flag
134+
*test.test_json.test_tool.TestTool.test_stdin_stdout
135+
*test.test_json.test_unicode.TestCUnicode.test_big_unicode_decode
136+
*test.test_json.test_unicode.TestCUnicode.test_big_unicode_encode
137+
*test.test_json.test_unicode.TestCUnicode.test_bytes_decode
138+
*test.test_json.test_unicode.TestCUnicode.test_bytes_encode
139+
*test.test_json.test_unicode.TestCUnicode.test_encoding3
140+
*test.test_json.test_unicode.TestCUnicode.test_encoding4
141+
*test.test_json.test_unicode.TestCUnicode.test_encoding5
142+
*test.test_json.test_unicode.TestCUnicode.test_encoding6
143+
*test.test_json.test_unicode.TestCUnicode.test_object_pairs_hook_with_unicode
144+
*test.test_json.test_unicode.TestCUnicode.test_unicode_decode
145+
*test.test_json.test_unicode.TestCUnicode.test_unicode_preservation
146+
*test.test_json.test_unicode.TestPyUnicode.test_big_unicode_decode
147+
*test.test_json.test_unicode.TestPyUnicode.test_big_unicode_encode
148+
*test.test_json.test_unicode.TestPyUnicode.test_bytes_decode
149+
*test.test_json.test_unicode.TestPyUnicode.test_bytes_encode
150+
*test.test_json.test_unicode.TestPyUnicode.test_encoding3
151+
*test.test_json.test_unicode.TestPyUnicode.test_encoding4
152+
*test.test_json.test_unicode.TestPyUnicode.test_encoding5
153+
*test.test_json.test_unicode.TestPyUnicode.test_encoding6
154+
*test.test_json.test_unicode.TestPyUnicode.test_object_pairs_hook_with_unicode
155+
*test.test_json.test_unicode.TestPyUnicode.test_unicode_decode
156+
*test.test_json.test_unicode.TestPyUnicode.test_unicode_preservation

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixSubprocessModuleBuiltins.java

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,9 @@
4040
*/
4141
package com.oracle.graal.python.builtins.modules;
4242

43-
import java.io.File;
4443
import java.io.IOException;
45-
import java.lang.ProcessBuilder.Redirect;
4644
import java.nio.ByteBuffer;
4745
import java.nio.channels.Channel;
48-
import java.nio.channels.Channels;
4946
import java.nio.channels.WritableByteChannel;
5047
import java.util.ArrayList;
5148
import java.util.HashMap;
@@ -76,6 +73,7 @@
7673
import com.oracle.graal.python.runtime.PythonContext;
7774
import com.oracle.graal.python.runtime.PythonOptions;
7875
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
76+
import com.oracle.graal.python.runtime.ProcessWrapper;
7977
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
8078
import com.oracle.truffle.api.TruffleFile;
8179
import com.oracle.truffle.api.TruffleLanguage.Env;
@@ -86,7 +84,9 @@
8684
import com.oracle.truffle.api.dsl.Specialization;
8785
import com.oracle.truffle.api.frame.VirtualFrame;
8886
import com.oracle.truffle.api.interop.UnsupportedMessageException;
87+
import com.oracle.truffle.api.io.TruffleProcessBuilder;
8988
import com.oracle.truffle.api.library.CachedLibrary;
89+
import org.graalvm.polyglot.io.ProcessHandler.Redirect;
9090

9191
@CoreFunctions(defineModule = "_posixsubprocess")
9292
public class PosixSubprocessModuleBuiltins extends PythonBuiltins {
@@ -142,11 +142,9 @@ private synchronized int forkExec(PList args, PList execList, @SuppressWarnings(
142142
}
143143
}
144144

145-
File cwdFile;
146145
Env truffleEnv = context.getEnv();
147-
if (getSafeTruffleFile(truffleEnv, cwd).exists()) {
148-
cwdFile = new File(cwd);
149-
} else {
146+
TruffleFile cwdFile = getSafeTruffleFile(truffleEnv, cwd);
147+
if (!cwdFile.exists()) {
150148
throw raise(PythonBuiltinClassType.OSError, ErrorMessages.WORK_DIR_NOT_ACCESSIBLE, cwd);
151149
}
152150

@@ -232,11 +230,11 @@ private synchronized int forkExec(PList args, PList execList, @SuppressWarnings(
232230

233231
// Tries executing given arguments, throws IOException if the executable cannot be executed,
234232
// any other error is handled here
235-
private int exec(ArrayList<String> argStrings, File cwd, Map<String, String> env,
233+
private int exec(ArrayList<String> argStrings, TruffleFile cwd, Map<String, String> env,
236234
int p2cwrite, int p2cread, int c2pwrite, int c2pread,
237235
int errwrite, int errpipe_write, PosixResources resources, int errread) throws IOException {
238236
LOGGER.finest(() -> "_posixsubprocess.fork_exec trying to exec: " + String.join(" ", argStrings));
239-
ProcessBuilder pb = new ProcessBuilder(argStrings);
237+
TruffleProcessBuilder pb = getContext().getEnv().newProcessBuilder(argStrings.toArray(new String[argStrings.size()]));
240238
if (p2cread != -1 && p2cwrite != -1) {
241239
pb.redirectInput(Redirect.PIPE);
242240
} else {
@@ -260,22 +258,22 @@ private int exec(ArrayList<String> argStrings, File cwd, Map<String, String> env
260258
}
261259

262260
pb.directory(cwd);
263-
pb.environment().putAll(env);
261+
pb.environment(env);
264262

265-
Process process = pb.start();
263+
ProcessWrapper process = new ProcessWrapper(pb.start(), p2cwrite != -1, c2pread != 1, errread != -1);
266264
try {
267265
if (p2cwrite != -1) {
268266
// user code is expected to close the unused ends of the pipes
269267
resources.getFileChannel(p2cwrite).close();
270-
resources.fdopen(p2cwrite, Channels.newChannel(process.getOutputStream()));
268+
resources.fdopen(p2cwrite, process.getOutputChannel());
271269
}
272270
if (c2pread != -1) {
273271
resources.getFileChannel(c2pread).close();
274-
resources.fdopen(c2pread, Channels.newChannel(process.getInputStream()));
272+
resources.fdopen(c2pread, process.getInputChannel());
275273
}
276274
if (errread != -1) {
277275
resources.getFileChannel(errread).close();
278-
resources.fdopen(errread, Channels.newChannel(process.getErrorStream()));
276+
resources.fdopen(errread, process.getErrorChannel());
279277
}
280278
} catch (IOException ex) {
281279
// We only want to rethrow the IOException that may come out of pb.start()

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SelectModuleBuiltins.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
6868
import com.oracle.graal.python.runtime.exception.PythonErrorType;
6969
import com.oracle.graal.python.runtime.sequence.PSequence;
70-
import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage;
7170
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
7271
import com.oracle.truffle.api.CompilerDirectives.ValueType;
7372
import com.oracle.truffle.api.dsl.Cached;
@@ -226,12 +225,13 @@ private ChannelFD[] seq2set(VirtualFrame frame, Object sequence, PythonObjectLib
226225
PSequence pSequence = constructListNode.execute(sequence);
227226

228227
for (int i = 0; i < len; i++) {
229-
int fd = itemLib.asFileDescriptorWithState(callGetItemNode.executeObject(frame, pSequence, i), threadState);
228+
Object pythonObject = callGetItemNode.executeObject(frame, pSequence, i);
229+
int fd = itemLib.asFileDescriptorWithState(pythonObject, threadState);
230230
Channel fileChannel = getContext().getResources().getFileChannel(fd);
231231
if (!(fileChannel instanceof SelectableChannel)) {
232232
throw NonSelectableChannel.INSTANCE;
233233
}
234-
result[i] = new ChannelFD(fd, (SelectableChannel) fileChannel);
234+
result[i] = new ChannelFD(pythonObject, (SelectableChannel) fileChannel);
235235
}
236236
return result;
237237
}
@@ -243,13 +243,13 @@ private PList toList(ChannelFD[] arr) {
243243
cnt++;
244244
}
245245
}
246-
int[] fds = new int[cnt];
246+
Object[] fds = new Object[cnt];
247247
for (ChannelFD channelFD : arr) {
248248
if (channelFD != null) {
249-
fds[fds.length - (cnt--)] = channelFD.fd;
249+
fds[fds.length - (cnt--)] = channelFD.pythonObject;
250250
}
251251
}
252-
return factory().createList(new IntSequenceStorage(fds));
252+
return factory().createList(fds);
253253
}
254254

255255
static LookupAndCallBinaryNode createGetItem() {
@@ -258,11 +258,11 @@ static LookupAndCallBinaryNode createGetItem() {
258258

259259
@ValueType
260260
private static final class ChannelFD {
261-
private final int fd;
261+
private final Object pythonObject;
262262
private final SelectableChannel channel;
263263

264-
private ChannelFD(int fd, SelectableChannel channel) {
265-
this.fd = fd;
264+
private ChannelFD(Object pythonObject, SelectableChannel channel) {
265+
this.pythonObject = pythonObject;
266266
this.channel = channel;
267267
}
268268
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/exception/OSErrorEnum.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,9 @@ private static String getReason(FileSystemException e) {
312312

313313
@TruffleBoundary
314314
private static OSErrorEnum tryFindErrnoFromMessage(Exception e) {
315+
if (e.getMessage().contains("Broken pipe")) {
316+
return OSErrorEnum.EPIPE;
317+
}
315318
Matcher m = ERRNO_PATTERN.matcher(e.getMessage());
316319
if (m.find()) {
317320
return fromNumber(Integer.parseInt(m.group(1)));

0 commit comments

Comments
 (0)