Skip to content

Commit 5bfd3fa

Browse files
msimacekmsimacek
committed
[GR-58925] Support unbuffered standard output
PullRequest: graalpython/3507
2 parents 0967368 + 7272662 commit 5bfd3fa

File tree

9 files changed

+85
-93
lines changed

9 files changed

+85
-93
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_cmd_line.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_site_flag
1919
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_stdin_readline
2020
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_unbuffered_input
21+
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_unbuffered_output
2122
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_unmached_quote
2223
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_usage
2324
*graalpython.lib-python.3.test.test_cmd_line.CmdLineTest.test_verbose

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.SyntaxError;
3131
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_FLUSH;
3232
import static com.oracle.graal.python.builtins.modules.io.IONodes.T_WRITE;
33+
import static com.oracle.graal.python.builtins.objects.PNone.NONE;
3334
import static com.oracle.graal.python.builtins.objects.PNone.NO_VALUE;
3435
import static com.oracle.graal.python.builtins.objects.PNotImplemented.NOT_IMPLEMENTED;
3536
import static com.oracle.graal.python.compiler.RaisePythonExceptionErrorCallback.raiseSyntaxError;
@@ -1884,6 +1885,10 @@ PNone printNoKeywords(VirtualFrame frame, Object[] values, PNone sep, PNone end,
18841885
@Shared("callFlush") @Cached PyObjectCallMethodObjArgs callFlush,
18851886
@Shared("strNode") @Cached PyObjectStrAsObjectNode strNode) {
18861887
Object stdout = getStdout();
1888+
// Allowed when stdout is not connected
1889+
if (stdout == NONE) {
1890+
return NONE;
1891+
}
18871892
return printAllGiven(frame, values, T_SPACE, T_NEWLINE, stdout, false, inliningTarget, getWriteMethod, callWrite, callFlush, strNode);
18881893
}
18891894

@@ -1941,6 +1946,10 @@ PNone printGeneric(VirtualFrame frame, Object[] values, Object sepIn, Object end
19411946
Object file;
19421947
if (fileIn instanceof PNone) {
19431948
file = getStdout();
1949+
// Allowed when stdout is not connected
1950+
if (file == NONE) {
1951+
return NONE;
1952+
}
19441953
} else {
19451954
file = fileIn;
19461955
}
@@ -1973,7 +1982,7 @@ private Object getStdout() {
19731982
readStdout = insert(ReadAttributeFromObjectNode.create());
19741983
}
19751984
Object stdout = readStdout.execute(sys, T_STDOUT);
1976-
if (stdout instanceof PNone) {
1985+
if (stdout == NO_VALUE) {
19771986
CompilerDirectives.transferToInterpreterAndInvalidate();
19781987
throw PRaiseNode.raiseUncached(this, RuntimeError, ErrorMessages.LOST_SYSSTDOUT);
19791988
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java

Lines changed: 36 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -583,21 +583,11 @@ public void initialize(Python3Core core) {
583583
final TruffleString gmultiarch = cat(PythonUtils.getPythonArch(), T_DASH, os.getName());
584584
addBuiltinConstant("__gmultiarch", gmultiarch);
585585

586-
PFileIO stdin = factory.createFileIO(PythonBuiltinClassType.PFileIO);
587-
FileIOBuiltins.FileIOInit.internalInit(stdin, toTruffleStringUncached("<stdin>"), 0, IOMode.R);
588-
addBuiltinConstant(T_STDIN, stdin);
589-
addBuiltinConstant(T___STDIN__, stdin);
586+
// Initialized later in postInitialize
587+
addBuiltinConstant(T_STDIN, PNone.NONE);
588+
addBuiltinConstant(T_STDOUT, PNone.NONE);
589+
addBuiltinConstant(T_STDERR, PNone.NONE);
590590

591-
PFileIO stdout = factory.createFileIO(PythonBuiltinClassType.PFileIO);
592-
FileIOBuiltins.FileIOInit.internalInit(stdout, toTruffleStringUncached("<stdout>"), 1, IOMode.W);
593-
addBuiltinConstant(T_STDOUT, stdout);
594-
addBuiltinConstant(T___STDOUT__, stdout);
595-
596-
PFileIO stderr = factory.createFileIO(PythonBuiltinClassType.PFileIO);
597-
stderr.setUTF8Write(true);
598-
FileIOBuiltins.FileIOInit.internalInit(stderr, toTruffleStringUncached("<stderr>"), 2, IOMode.W);
599-
addBuiltinConstant(T_STDERR, stderr);
600-
addBuiltinConstant(T___STDERR__, stderr);
601591
addBuiltinConstant("implementation", makeImplementation(factory, versionInfo, gmultiarch));
602592
addBuiltinConstant("hexversion", PythonLanguage.VERSION_HEX);
603593

@@ -773,37 +763,52 @@ public void postInitialize(Python3Core core) {
773763
}
774764

775765
@TruffleBoundary
776-
public void initStd(Python3Core core) {
766+
static void initStd(Python3Core core) {
777767
PythonObjectFactory factory = core.factory();
768+
PythonContext context = core.getContext();
778769

779770
// wrap std in/out/err
780771
GraalPythonModuleBuiltins gp = (GraalPythonModuleBuiltins) core.lookupBuiltinModule(T___GRAALPYTHON__).getBuiltins();
781772
TruffleString stdioEncoding = gp.getStdIOEncoding();
782773
TruffleString stdioError = gp.getStdIOError();
783-
Object posixSupport = core.getContext().getPosixSupport();
774+
Object posixSupport = context.getPosixSupport();
784775
PosixSupportLibrary posixLib = PosixSupportLibrary.getUncached();
785776
PythonModule sysModule = core.lookupBuiltinModule(T_SYS);
786777

787-
PBuffered reader = factory.createBufferedReader(PythonBuiltinClassType.PBufferedReader);
788-
BufferedReaderBuiltins.BufferedReaderInit.internalInit(reader, (PFileIO) getBuiltinConstant(T_STDIN), BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport,
789-
posixLib);
790-
setWrapper(T_STDIN, T___STDIN__, T_R, stdioEncoding, stdioError, reader, sysModule, factory);
778+
// Note that stdin is always buffered, this only applies to stdout and stderr
779+
boolean buffering = !context.getOption(PythonOptions.UnbufferedIO);
780+
781+
PFileIO stdinFileIO = factory.createFileIO(PythonBuiltinClassType.PFileIO);
782+
FileIOBuiltins.FileIOInit.internalInit(stdinFileIO, toTruffleStringUncached("<stdin>"), 0, IOMode.RB);
783+
PBuffered stdinBuffer = factory.createBufferedReader(PythonBuiltinClassType.PBufferedReader);
784+
BufferedReaderBuiltins.BufferedReaderInit.internalInit(stdinBuffer, stdinFileIO, BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport, posixLib);
785+
setWrapper(T_STDIN, T___STDIN__, T_R, stdioEncoding, stdioError, stdinBuffer, sysModule, factory, true);
786+
787+
PFileIO stdoutFileIO = factory.createFileIO(PythonBuiltinClassType.PFileIO);
788+
FileIOBuiltins.FileIOInit.internalInit(stdoutFileIO, toTruffleStringUncached("<stdout>"), 1, IOMode.WB);
789+
Object stdoutBuffer = createBufferedIO(buffering, factory, stdoutFileIO, posixSupport, posixLib);
790+
setWrapper(T_STDOUT, T___STDOUT__, T_W, stdioEncoding, stdioError, stdoutBuffer, sysModule, factory, buffering);
791791

792+
PFileIO stderr = factory.createFileIO(PythonBuiltinClassType.PFileIO);
793+
FileIOBuiltins.FileIOInit.internalInit(stderr, toTruffleStringUncached("<stderr>"), 2, IOMode.WB);
794+
Object stderrBuffer = createBufferedIO(buffering, factory, stderr, posixSupport, posixLib);
795+
setWrapper(T_STDERR, T___STDERR__, T_W, stdioEncoding, T_BACKSLASHREPLACE, stderrBuffer, sysModule, factory, buffering);
796+
}
797+
798+
private static Object createBufferedIO(boolean buffering, PythonObjectFactory factory, PFileIO fileIo, Object posixSupport, PosixSupportLibrary posixLib) {
799+
if (!buffering) {
800+
return fileIo;
801+
}
792802
PBuffered writer = factory.createBufferedWriter(PythonBuiltinClassType.PBufferedWriter);
793-
BufferedWriterBuiltins.BufferedWriterInit.internalInit(writer, (PFileIO) getBuiltinConstant(T_STDOUT), BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport,
794-
posixLib);
795-
setWrapper(T_STDOUT, T___STDOUT__, T_W, stdioEncoding, stdioError, writer, sysModule, factory);
796-
797-
writer = factory.createBufferedWriter(PythonBuiltinClassType.PBufferedWriter);
798-
BufferedWriterBuiltins.BufferedWriterInit.internalInit(writer, (PFileIO) getBuiltinConstant(T_STDERR), BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport,
799-
posixLib);
800-
setWrapper(T_STDERR, T___STDERR__, T_W, stdioEncoding, T_BACKSLASHREPLACE, writer, sysModule, factory);
803+
BufferedWriterBuiltins.BufferedWriterInit.internalInit(writer, fileIo, BufferedReaderBuiltins.DEFAULT_BUFFER_SIZE, factory, posixSupport, posixLib);
804+
return writer;
801805
}
802806

803-
private static PTextIO setWrapper(TruffleString name, TruffleString specialName, TruffleString mode, TruffleString encoding, TruffleString error, PBuffered buffered, PythonModule sysModule,
804-
PythonObjectFactory factory) {
807+
private static PTextIO setWrapper(TruffleString name, TruffleString specialName, TruffleString mode, TruffleString encoding, TruffleString error, Object buffer, PythonModule sysModule,
808+
PythonObjectFactory factory, boolean buffering) {
805809
PTextIO textIOWrapper = factory.createTextIO(PythonBuiltinClassType.PTextIOWrapper);
806-
TextIOWrapperInitNodeGen.getUncached().execute(null, null, textIOWrapper, buffered, encoding, error, PNone.NONE, true, true);
810+
TextIOWrapperInitNodeGen.getUncached().execute(null, null, textIOWrapper, buffer, encoding, error, PNone.NONE,
811+
/* line_buffering */ buffering, /* write_through */ !buffering);
807812

808813
setAttribute(textIOWrapper, T_MODE, mode);
809814
setAttribute(sysModule, name, textIOWrapper);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/FileIOBuiltins.java

Lines changed: 27 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,7 @@
8989
import static com.oracle.graal.python.nodes.SpecialMethodNames.J___INIT__;
9090
import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REPR__;
9191
import static com.oracle.graal.python.nodes.StringLiterals.T_FALSE;
92-
import static com.oracle.graal.python.nodes.StringLiterals.T_STRICT;
9392
import static com.oracle.graal.python.nodes.StringLiterals.T_TRUE;
94-
import static com.oracle.graal.python.nodes.StringLiterals.T_UTF8;
9593
import static com.oracle.graal.python.runtime.PosixConstants.AT_FDCWD;
9694
import static com.oracle.graal.python.runtime.PosixConstants.O_APPEND;
9795
import static com.oracle.graal.python.runtime.PosixConstants.O_CREAT;
@@ -113,7 +111,6 @@
113111
import com.oracle.graal.python.builtins.CoreFunctions;
114112
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
115113
import com.oracle.graal.python.builtins.PythonBuiltins;
116-
import com.oracle.graal.python.builtins.modules.CodecsModuleBuiltins;
117114
import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins;
118115
import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.FtruncateNode;
119116
import com.oracle.graal.python.builtins.modules.PosixModuleBuiltins.LseekNode;
@@ -122,7 +119,6 @@
122119
import com.oracle.graal.python.builtins.modules.io.IONodes.IOMode;
123120
import com.oracle.graal.python.builtins.objects.PNone;
124121
import com.oracle.graal.python.builtins.objects.buffer.PythonBufferAccessLibrary;
125-
import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
126122
import com.oracle.graal.python.builtins.objects.bytes.PBytes;
127123
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
128124
import com.oracle.graal.python.builtins.objects.exception.OSErrorEnum;
@@ -713,59 +709,44 @@ protected ArgumentClinicProvider getArgumentClinic() {
713709
}
714710
}
715711

716-
@Builtin(name = J_WRITE, minNumOfPositionalArgs = 2)
712+
@Builtin(name = J_WRITE, minNumOfPositionalArgs = 2, numOfPositionalOnlyArgs = 2, parameterNames = {"$self", "b"})
713+
@ArgumentClinic(name = "b", conversion = ArgumentClinic.ClinicConversion.ReadableBuffer)
717714
@GenerateNodeFactory
718-
public abstract static class WriteNode extends PythonBinaryBuiltinNode {
715+
public abstract static class WriteNode extends PythonBinaryClinicBuiltinNode {
719716

720-
@Specialization(guards = {"!self.isClosed()", "self.isWritable()"})
721-
static Object write(VirtualFrame frame, PFileIO self, Object data,
717+
@Specialization(limit = "3")
718+
static Object write(VirtualFrame frame, PFileIO self, Object buffer,
722719
@Bind("this") Node inliningTarget,
723-
@Cached GetBytesToWriteNode getBytesToWriteNode,
720+
@CachedLibrary("buffer") PythonBufferAccessLibrary bufferLib,
724721
@CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib,
725722
@Cached InlinedBranchProfile errorProfile,
726723
@Cached GilNode gil,
727-
@Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) {
728-
byte[] bytes = getBytesToWriteNode.execute(frame, inliningTarget, self, data);
724+
@Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode,
725+
@Cached PRaiseNode.Lazy raiseNode) {
729726
try {
730-
return PosixModuleBuiltins.WriteNode.write(self.getFD(), bytes, bytes.length, inliningTarget, posixLib, errorProfile, gil);
731-
} catch (PosixException e) {
732-
if (e.getErrorCode() == EAGAIN.getNumber()) {
733-
return PNone.NONE;
727+
if (self.isClosed()) {
728+
throw raiseNode.get(inliningTarget).raise(ValueError, IO_CLOSED);
734729
}
735-
errorProfile.enter(inliningTarget);
736-
throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e);
730+
if (!self.isWritable()) {
731+
throw raiseNode.get(inliningTarget).raise(IOUnsupportedOperation, FILE_NOT_OPEN_FOR_S, "writing");
732+
}
733+
try {
734+
return PosixModuleBuiltins.WriteNode.write(self.getFD(), bufferLib.getInternalOrCopiedByteArray(buffer), bufferLib.getBufferLength(buffer),
735+
inliningTarget, posixLib, errorProfile, gil);
736+
} catch (PosixException e) {
737+
if (e.getErrorCode() == EAGAIN.getNumber()) {
738+
return PNone.NONE;
739+
}
740+
throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e);
741+
}
742+
} finally {
743+
bufferLib.release(buffer);
737744
}
738745
}
739746

740-
@Specialization(guards = {"!self.isClosed()", "!self.isWritable()"})
741-
static Object notWritable(@SuppressWarnings("unused") PFileIO self, @SuppressWarnings("unused") Object buf,
742-
@Shared @Cached PRaiseNode raiseNode) {
743-
throw raiseNode.raise(IOUnsupportedOperation, FILE_NOT_OPEN_FOR_S, "writing");
744-
}
745-
746-
@Specialization(guards = "self.isClosed()")
747-
static Object closedError(@SuppressWarnings("unused") PFileIO self, @SuppressWarnings("unused") Object buf,
748-
@Shared @Cached PRaiseNode raiseNode) {
749-
throw raiseNode.raise(ValueError, IO_CLOSED);
750-
}
751-
}
752-
753-
@GenerateInline
754-
@GenerateCached(false)
755-
abstract static class GetBytesToWriteNode extends Node {
756-
abstract byte[] execute(VirtualFrame frame, Node inliningTarget, PFileIO self, Object data);
757-
758-
@Specialization(guards = "!self.isUTF8Write()")
759-
static byte[] doBytes(VirtualFrame frame, @SuppressWarnings("unused") PFileIO self, Object data,
760-
@Cached(inline = false) BytesNodes.ToBytesNode toBytes) {
761-
return toBytes.execute(frame, data);
762-
}
763-
764-
@Specialization(guards = "self.isUTF8Write()")
765-
static byte[] doUtf8(Node inliningTarget, @SuppressWarnings("unused") PFileIO self, Object data,
766-
@Cached(inline = false) CodecsModuleBuiltins.CodecsEncodeToJavaBytesNode encode,
767-
@Cached CastToTruffleStringNode castStr) {
768-
return encode.execute(castStr.execute(inliningTarget, data), T_UTF8, T_STRICT);
747+
@Override
748+
protected ArgumentClinicProvider getArgumentClinic() {
749+
return FileIOBuiltinsClinicProviders.WriteNodeClinicProviderGen.INSTANCE;
769750
}
770751
}
771752

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/IONodes.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -214,12 +214,14 @@ public class IONodes {
214214
public static final TruffleString T_R = tsLiteral("r");
215215
public static final TruffleString T_W = tsLiteral("w");
216216
public static final TruffleString T_RB = tsLiteral("rb");
217+
public static final TruffleString T_WB = tsLiteral("wb");
217218

218219
@CompilerDirectives.ValueType
219220
public static final class IOMode {
220221
public static final IOMode R = new IOMode(T_R, true, false, false, 1);
221222
public static final IOMode W = new IOMode(T_W, false, true, false, 1);
222223
public static final IOMode RB = new IOMode(T_RB, true, false, true, 1);
224+
public static final IOMode WB = new IOMode(T_WB, false, true, true, 1);
223225

224226
boolean creating;
225227
boolean reading;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/PFileIO.java

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -62,8 +62,6 @@ public final class PFileIO extends PythonBuiltinObject {
6262
boolean finalizing;
6363
private int blksize;
6464

65-
private boolean utf8Write;
66-
6765
public PFileIO(Object cls, Shape instanceShape) {
6866
super(cls, instanceShape);
6967
this.fd = null;
@@ -166,14 +164,6 @@ public int getBlksize() {
166164
public void setBlksize(int blksize) {
167165
this.blksize = blksize;
168166
}
169-
170-
public boolean isUTF8Write() {
171-
return utf8Write;
172-
}
173-
174-
public void setUTF8Write(boolean utf8Write) {
175-
this.utf8Write = utf8Write;
176-
}
177167
}
178168

179169
@ValueType

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/io/TextIOWrapperNodes.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -881,7 +881,6 @@ static void init(VirtualFrame frame, Node inliningTarget, PTextIO self, Object b
881881
if (((PBuffered) buffer).isFastClosedChecks()) {
882882
PFileIO f = ((PBuffered) buffer).getFileIORaw();
883883
self.setFileIO(f);
884-
f.setUTF8Write(false);
885884
}
886885
}
887886

‎graalpython/lib-graalpython/patches/meson-python/metadata.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
[[rules]]
2-
# Backport pypa/packaging PR for fixing graalpy architecture
2+
# Upstreamed
3+
version = '<= 0.16.0'
4+
install-priority = 0
35
patch = 'meson-python.patch'
46
license = 'MIT'

‎graalpython/lib-graalpython/patches/pybind11/metadata.toml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
[[rules]]
2+
# Upstreamed
3+
install-priority = 0
24
# Note: This patch is also inlined in torch
3-
version = '>= 2.11.0'
5+
version = '>= 2.11.0, <= 2.13.6'
46
patch = 'pybind11-2.11.patch'
57
license = 'BSD-3-Clause'
68

79
[[rules]]
10+
install-priority = 0
811
# Note: This patch file is also used directly outside of pip during onnxruntime build
912
# Note: This patch file is also used directly outside of pip during tensorflow build
1013
# Note: This patch is also inlined in torch

0 commit comments

Comments
 (0)