From 58d211ba3a9d494c2ce89337dc60e0c24773856e Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 19:26:59 +0000 Subject: [PATCH 01/13] mitogen/parent: Fix typo Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 6e30b1c62..15c8c5285 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1417,7 +1417,7 @@ def __repr__(self): # w: write side of core_src FD. # C: the decompressed core source. - # Final os.close(STDOUT_FILENO) to avoid --py-debug build corrupting stream with + # Final os.close(STDERR_FILENO) to avoid --py-debug build corrupting stream with # "[1234 refs]" during exit. @staticmethod def _first_stage(): From ead4a6da5077f28cd59a1d905ac8a0d9c13dfcfc Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:58:40 +0000 Subject: [PATCH 02/13] first_stage_test: Refactor the test This makes it easier to add more tests and the test description is now used by the test runner. Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 40 +++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 2576ec14d..0c2ed3fb5 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,5 +1,3 @@ -import subprocess - import mitogen.core import mitogen.parent from mitogen.core import b @@ -16,29 +14,39 @@ class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # * 2.7 starting 3.x # * 3.x starting 2.7 - def test_valid_syntax(self): + def setUp(self): + super(CommandLineTest, self).setUp() options = mitogen.parent.Options(max_message_size=123) conn = mitogen.parent.Connection(options, self.router) conn.context = mitogen.core.Context(None, 123) - args = conn.get_boot_command() + self.args = conn.get_boot_command() + self.preamble = conn.get_preamble() + self.conn = conn + + def test_valid_syntax(self): + """Test valid syntax + + The boot command should write an ECO marker to stdout, read the + preamble from stdin, then execute it. + + This test attaches /dev/zero to stdin to create a specific failure - # The boot command should write an ECO marker to stdout, read the - # preamble from stdin, then execute it. + 1. Fork child reads bytes of NUL (`b'\0'`) + 2. Fork child crashes (trying to decompress the junk data) + 3. Fork child's file descriptors (write pipes) are closed by the OS + 4. Fork parent does `dup(, )` and `exec()` + 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) + 6. Python runs `''` (a valid script) and exits with success - # This test attaches /dev/zero to stdin to create a specific failure - # 1. Fork child reads bytes of NUL (`b'\0'`) - # 2. Fork child crashes (trying to decompress the junk data) - # 3. Fork child's file descriptors (write pipes) are closed by the OS - # 4. Fork parent does `dup(, )` and `exec()` - # 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) - # 6. Python runs `''` (a valid script) and exits with success + """ fp = open("/dev/zero", "r") try: - proc = subprocess.Popen(args, + proc = testlib.subprocess.Popen( + self.args, stdin=fp, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=testlib.subprocess.PIPE, + stderr=testlib.subprocess.PIPE, ) stdout, stderr = proc.communicate() self.assertEqual(0, proc.returncode) From 69919995947acc31bc050a2f5d2bb37853c0af58 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:59:51 +0000 Subject: [PATCH 03/13] first_stage_test: Add more tests + test_stdin_non_blocking + test_stdin_blocking Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 124 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 0c2ed3fb5..9e9217945 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,3 +1,7 @@ +import fcntl +import functools +import operator + import mitogen.core import mitogen.parent from mitogen.core import b @@ -5,6 +9,126 @@ import testlib +def own_create_child(args, blocking, pipe_size=None, preexec_fn=None, pass_stderr=True): + """ + Create a child process whose stdin/stdout/stderr is connected to a pipe. + + :param list args: + Program argument vector. + :param bool blocking: + If :data:`True`, the pipes use blocking IO, otherwise non-blocking. + :param int pipe_size: + If not :data:`None`, use the values as the pipe size. + :param function preexec_fn: + If not :data:`None`, a function to run within the post-fork child + before executing the target program. + :returns: + :class:`PopenProcess` instance. + """ + parent_rfp, child_wfp = mitogen.core.pipe(blocking=blocking) + child_rfp, parent_wfp = mitogen.core.pipe(blocking=blocking) + stderr_r, stderr = mitogen.core.pipe(blocking=blocking) + mitogen.core.set_cloexec(stderr_r.fileno()) + if pipe_size is not None: + fcntl.fcntl(parent_rfp.fileno(), fcntl.F_SETPIPE_SZ, pipe_size) + fcntl.fcntl(child_rfp.fileno(), fcntl.F_SETPIPE_SZ, pipe_size) + fcntl.fcntl(stderr_r.fileno(), fcntl.F_SETPIPE_SZ, pipe_size) + assert fcntl.fcntl(parent_rfp.fileno(), fcntl.F_GETPIPE_SZ) == pipe_size + assert fcntl.fcntl(child_rfp.fileno(), fcntl.F_GETPIPE_SZ) == pipe_size + assert fcntl.fcntl(stderr_r.fileno(), fcntl.F_GETPIPE_SZ) == pipe_size + + try: + proc = testlib.subprocess.Popen( + args=args, + stdin=child_rfp, + stdout=child_wfp, + stderr=stderr, + preexec_fn=preexec_fn, + ) + except Exception: + child_rfp.close() + child_wfp.close() + parent_rfp.close() + parent_wfp.close() + stderr_r.close() + stderr.close() + raise + + child_rfp.close() + child_wfp.close() + stderr.close() + # Only used to create a specific test scenario! + if not pass_stderr: + stderr_r.close() + stderr_r = None + return mitogen.parent.PopenProcess( + proc=proc, + stdin=parent_wfp, + stdout=parent_rfp, + stderr=stderr_r, + ) + + +class DummyConnectionBlocking(mitogen.parent.Connection): + """Dummy blocking IO connection""" + + pipe_size = 4096 if getattr(fcntl, "F_SETPIPE_SZ", None) else None + create_child = staticmethod( + functools.partial(own_create_child, blocking=True, pipe_size=pipe_size) + ) + name_prefix = "dummy_blocking" + + +class DummyConnectionNonBlocking(mitogen.parent.Connection): + """Dummy non-blocking IO connection""" + + pipe_size = 4096 if getattr(fcntl, "F_SETPIPE_SZ", None) else None + create_child = staticmethod( + functools.partial(own_create_child, blocking=False, pipe_size=pipe_size) + ) + name_prefix = "dummy_non_blocking" + + +class ConnectionTest(testlib.RouterMixin, testlib.TestCase): + def test_non_blocking_stdin(self): + """Test that first stage works with non-blocking STDIN + + The boot command should read the preamble from STDIN, write all ECO + markers to STDOUT, and then execute the preamble. + + This test writes the complete preamble to non-blocking STDIN. + + 1. Fork child reads from non-blocking STDIN + 2. Fork child writes all data as expected by the protocol. + 3. A context call works as expected. + + """ + log = testlib.LogCapturer() + log.start() + ctx = self.router._connect(DummyConnectionNonBlocking, connect_timeout=0.5) + self.assertEqual(3, ctx.call(operator.add, 1, 2)) + logs = log.stop() + + def test_blocking_stdin(self): + """Test that first stage works with blocking STDIN + + The boot command should read the preamble from STDIN, write all ECO + markers to STDOUT, and then execute the preamble. + + This test writes the complete preamble to blocking STDIN. + + 1. Fork child reads from blocking STDIN + 2. Fork child writes all data as expected by the protocol. + 3. A context call works as expected. + + """ + log = testlib.LogCapturer() + log.start() + ctx = self.router._connect(DummyConnectionBlocking, connect_timeout=0.5) + self.assertEqual(3, ctx.call(operator.add, 1, 2)) + logs = log.stop() + + class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # Ensure this version of Python produces a command line that is sufficient # to bootstrap this version of Python. From e9f7d34bf0befd4472e4d18868e2fe180eedb4cf Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 20:12:10 +0000 Subject: [PATCH 04/13] first_stage_test: Add test_premature_eof test Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 9e9217945..4387a0d6a 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -182,3 +182,47 @@ def test_valid_syntax(self): ) finally: fp.close() + + def test_premature_eof(self): + """The boot command should write an ECO marker to stdout, read the + preamble from stdin, then execute it. + + This test writes some data to STDIN and closes it then to create an + EOF situation. + 1. Fork child tries to read from STDIN, but stops as EOF is received. + 2. Fork child crashes (trying to decompress the junk data) + 3. Fork child's file descriptors (write pipes) are closed by the OS + 4. Fork parent does `dup(, )` and `exec()` + 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) + 6. Python runs `''` (a valid script) and exits with success""" + + proc = testlib.subprocess.Popen( + args=self.args, + stdout=testlib.subprocess.PIPE, + stderr=testlib.subprocess.PIPE, + stdin=testlib.subprocess.PIPE, + ) + + # Do not send all of the data from the preamble + proc.stdin.write(self.preamble[:-128]) + proc.stdin.flush() + proc.stdin.close() + try: + returncode = proc.wait(timeout=10) + except testlib.subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) + self.fail("First stage did not handle EOF on STDIN") + try: + self.assertEqual(0, returncode) + self.assertEqual( + proc.stdout.read(), + mitogen.parent.BootstrapProtocol.EC0_MARKER + b("\n"), + ) + self.assertIn( + b("Error -5 while decompressing data"), + proc.stderr.read(), + ) + finally: + proc.stdout.close() + proc.stderr.close() From ab93ea76563885db93a6fda31ed4c29cec0150d6 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:57:48 +0000 Subject: [PATCH 05/13] mitogen: first_stage: Break the while loop in case of EOF The current implementation can cause an infinite loop, leading to a process that hangs and consumes 100% CPU. This occurs because the EOF condition is not handled properly, resulting in repeated select(...) and read(...) calls. The fix is to properly handle the EOF condition and break out of the loop when it occurs. -SSH command size: 822 +SSH command size: 838 -mitogen.parent 98746 96.4KiB 51215 50.0KiB 51.9% 12922 12.6KiB 13.1% +mitogen.parent 98827 96.5KiB 51219 50.0KiB 51.8% 12942 12.6KiB 13.1% Fixes: https://github.com/mitogen-hq/mitogen/issues/1348 Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 15c8c5285..ae2413ee2 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1416,6 +1416,8 @@ def __repr__(self): # r: read side of core_src FD. # w: write side of core_src FD. # C: the decompressed core source. + # n: size of the compressed core source to be read + # V: data chunk # Final os.close(STDERR_FILENO) to avoid --py-debug build corrupting stream with # "[1234 refs]" during exit. @@ -1437,8 +1439,8 @@ def _first_stage(): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2]) os.write(1,'MITO000\n'.encode()) - C=''.encode() - while int(sys.argv[3])-len(C)and select.select([0],[],[]):C+=os.read(0,int(sys.argv[3])-len(C)) + n=int(sys.argv[3]);C=''.encode();V='V' + while n>len(C) and V:select.select([0],[],[]);V=os.read(0,n-len(C));C+=V C=zlib.decompress(C) f=os.fdopen(W,'wb',0) f.write(C) From 5e4bd9540da0e1bba97fa1111ef3dc58beea6ba5 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 20:17:36 +0000 Subject: [PATCH 06/13] first_stage_test: Add test_timeout_error Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 4387a0d6a..a452e428e 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -226,3 +226,42 @@ def test_premature_eof(self): finally: proc.stdout.close() proc.stderr.close() + + def test_timeout_error(self): + """The boot command should write an ECO marker to stdout, try to read + the preamble from stdin, then fail with an TimeoutError as nothing has + been written. + + This test writes no data to STDIN of the fork child to enforce a time out. + 1. Fork child tries to read from STDIN, but runs into the timeout + 2. Fork child raises TimeoutError + 3. Fork child's file descriptors (write pipes) are closed by the OS + 4. Fork parent does `dup(, )` and `exec()` + 5. Python reads `b''` (i.e. EOF) from stdin (a closed pipe) + 6. Python runs `''` (a valid script) and exits with success + """ + + proc = testlib.subprocess.Popen( + args=self.args, + stdout=testlib.subprocess.PIPE, + stderr=testlib.subprocess.PIPE, + close_fds=True, + ) + try: + returncode = proc.wait(timeout=12) + except testlib.subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) + self.fail("Timeout situation was not recognized") + else: + stdout = proc.stdout.read() + stderr = proc.stderr.read() + finally: + proc.stdout.close() + proc.stderr.close() + self.assertEqual(0, returncode) + self.assertEqual(stdout, mitogen.parent.BootstrapProtocol.EC0_MARKER + b("\n")) + self.assertIn( + b(""), + stderr, + ) From 9234a4070e1a3c2138e46bed0aba72d00449a87e Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Mon, 8 Dec 2025 09:43:39 +0000 Subject: [PATCH 07/13] mitogen: _first_stage: Add timeout handling Do not wait/block forever for data to be read. Add a test for this. The test can be run using the following command: PYTHONPATH=$(pwd)/tests:$PYTHONPATH python -m unittest -v tests.first_stage_test -SSH command size: 838 +SSH command size: 894 Original Minimized Compressed -mitogen.parent 98827 96.5KiB 51219 50.0KiB 51.8% 12942 12.6KiB 13.1% +mitogen.parent 99034 96.7KiB 51295 50.1KiB 51.8% 12970 12.7KiB 13.1% Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index ae2413ee2..6424434f4 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1418,6 +1418,8 @@ def __repr__(self): # C: the decompressed core source. # n: size of the compressed core source to be read # V: data chunk + # rl: list of FDs ready for reading + # _: throw away variable # Final os.close(STDERR_FILENO) to avoid --py-debug build corrupting stream with # "[1234 refs]" during exit. @@ -1439,8 +1441,15 @@ def _first_stage(): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2]) os.write(1,'MITO000\n'.encode()) - n=int(sys.argv[3]);C=''.encode();V='V' - while n>len(C) and V:select.select([0],[],[]);V=os.read(0,n-len(C));C+=V + n=int(sys.argv[3]) + C=''.encode() + V='V' + while n>len(C) and V: + rl,_,_=select.select([0],[],[],10) + if not rl: + sys.exit(1) + V=os.read(0,n-len(C)) + C+=V C=zlib.decompress(C) f=os.fdopen(W,'wb',0) f.write(C) From 67664043f7a2683c1edadfa97dd369848f23fa94 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 09:54:33 +0000 Subject: [PATCH 08/13] mitogen/parent: Make the timeout configurable -SSH command size: 894 +SSH command size: 905 Original Minimized Compressed -mitogen.parent 99034 96.7KiB 51295 50.1KiB 51.8% 12970 12.7KiB 13.1% +mitogen.parent 99212 96.9KiB 51385 50.2KiB 51.8% 12999 12.7KiB 13.1% Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 6424434f4..0b9260ec3 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1378,6 +1378,9 @@ class Connection(object): #: user. exception = None + #: First stage select timeout in seconds. + _first_stage_select_timeout = 10 + #: Extra text appended to :class:`EofError` if that exception is raised on #: a failed connection attempt. May be used in subclasses to hint at common #: problems with a particular connection method. @@ -1419,6 +1422,7 @@ def __repr__(self): # n: size of the compressed core source to be read # V: data chunk # rl: list of FDs ready for reading + # t: timeout value in seconds # _: throw away variable # Final os.close(STDERR_FILENO) to avoid --py-debug build corrupting stream with @@ -1442,10 +1446,11 @@ def _first_stage(): os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2]) os.write(1,'MITO000\n'.encode()) n=int(sys.argv[3]) + t=float(sys.argv[4]) C=''.encode() V='V' while n>len(C) and V: - rl,_,_=select.select([0],[],[],10) + rl,_,_=select.select([0],[],[],t) if not rl: sys.exit(1) V=os.read(0,n-len(C)) @@ -1495,6 +1500,7 @@ def get_boot_command(self): encoded.decode(), self.options.remote_name, str(len(self.get_preamble())), + str(self._first_stage_select_timeout), ] def get_econtext_config(self): From 7366652b583b286694976eb88d06d43d051b0c33 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 20:24:21 +0000 Subject: [PATCH 09/13] first_stage_test: test_timeout_error: Use smaller timeout Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index a452e428e..e08878836 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -241,14 +241,18 @@ def test_timeout_error(self): 6. Python runs `''` (a valid script) and exits with success """ + # We do not want to wait the default of 10s, change it to 0.1s + self.conn._first_stage_select_timeout = 0.1 + args = self.conn.get_boot_command() + proc = testlib.subprocess.Popen( - args=self.args, + args=args, stdout=testlib.subprocess.PIPE, stderr=testlib.subprocess.PIPE, close_fds=True, ) try: - returncode = proc.wait(timeout=12) + returncode = proc.wait(timeout=3) except testlib.subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=3) From e7b9a17f22dd1dfdb06b0fc81ad1ea860d803105 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 20:12:40 +0000 Subject: [PATCH 10/13] first_stage_test: Add test_closed_stdin and test_closed_stdout Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 82 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index e08878836..19bdc89b1 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,6 +1,7 @@ import fcntl import functools import operator +import os import mitogen.core import mitogen.parent @@ -269,3 +270,84 @@ def test_timeout_error(self): b(""), stderr, ) + + def test_closed_stdin(self): + """This test closes STDIN of the child process. + + 1. The child process detects that STDIN is unavailable + 2. The child process terminates early with an OSError exception, and + reports the issue via exception printed on STDERR. + 3. The parent process correctly identifies this condition. + + """ + # We do not want to wait the default of 10s, change it to 0.1s + self.conn._first_stage_timeout = 0.1 + args = self.conn.get_boot_command() + + proc = testlib.subprocess.Popen( + args=args, + stdout=testlib.subprocess.PIPE, + stderr=testlib.subprocess.PIPE, + preexec_fn=lambda: os.close(0), + close_fds=True, + ) + try: + stdout, stderr = proc.communicate(timeout=12) + except testlib.subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) + self.fail("Closed STDIN situation was not recognized") + self.assertEqual(1, proc.returncode) + self.assertEqual(stdout, b"") + self.assertIn( + b("Bad file descriptor"), + stderr, + ) + + def test_closed_stdout(self): + """Test that first stage bails out if STDOUT is closed + + This test closes STDOUT of the child process. + + 1. The child process detects that STDOUT is unavailable + 2. The child process terminates early with an OSError exception, and + reports the issue via exception printed on STDERR. + 3. The parent process correctly identifies this condition. + + """ + + stdout_r, stdout_w = mitogen.core.pipe() + mitogen.core.set_cloexec(stdout_r.fileno()) + stderr_r, stderr_w = mitogen.core.pipe() + mitogen.core.set_cloexec(stderr_r.fileno()) + try: + proc = testlib.subprocess.Popen( + args=self.args, + stdout=stdout_w, + stderr=stderr_w, + preexec_fn=lambda: os.close(0), + ) + except Exception: + stdout_r.close() + stdout_w.close() + stderr_w.close() + stderr_r.close() + raise + stdout_w.close() + stderr_w.close() + try: + returncode = proc.wait(timeout=1) + except testlib.subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=3) + self.fail("Closed STDOUT situation was not detected") + else: + stderr = stderr_r.read() + finally: + stderr_r.close() + stdout_r.close() + self.assertEqual(1, returncode) + self.assertIn( + b("Bad file descriptor"), + stderr, + ) From 8abb6cdb5453a732708e7443badf85b7cb505140 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 19:36:52 +0000 Subject: [PATCH 11/13] mitogen/parent: Bail out if STDIN or STDOUT is closed Bail out if STDIN or STDOUT is closed/not available as it is used for the communication with the parent process. Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mitogen/parent.py b/mitogen/parent.py index 0b9260ec3..5349c6ef8 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1429,6 +1429,8 @@ def __repr__(self): # "[1234 refs]" during exit. @staticmethod def _first_stage(): + os.fstat(0) + os.fstat(1) R,W=os.pipe() r,w=os.pipe() if os.fork(): From 343f2c986e7fdae6b374d151a9ae5382f46489fa Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 20:13:00 +0000 Subject: [PATCH 12/13] first_stage_test: Add test_closed_stderr Signed-off-by: Marc Hartmayer --- tests/first_stage_test.py | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/first_stage_test.py b/tests/first_stage_test.py index 19bdc89b1..ee7475600 100644 --- a/tests/first_stage_test.py +++ b/tests/first_stage_test.py @@ -1,5 +1,6 @@ import fcntl import functools +import sys import operator import os @@ -90,6 +91,24 @@ class DummyConnectionNonBlocking(mitogen.parent.Connection): name_prefix = "dummy_non_blocking" +class DummyConnectionClosedStderr(mitogen.parent.Connection): + """Dummy closed stderr connection""" + + pipe_size = 4096 if getattr(fcntl, "F_SETPIPE_SZ", None) else None + create_child = staticmethod( + functools.partial( + own_create_child, + blocking=True, + pipe_size=pipe_size, + pass_stderr=False, + # `os.close(2)` does not work here as we use file objects in + # `create_child` and that would cause problems with Python2. + preexec_fn=lambda: sys.stderr.close(), + ) + ) + name_prefix = "dummy_closed_stderr" + + class ConnectionTest(testlib.RouterMixin, testlib.TestCase): def test_non_blocking_stdin(self): """Test that first stage works with non-blocking STDIN @@ -129,6 +148,32 @@ def test_blocking_stdin(self): self.assertEqual(3, ctx.call(operator.add, 1, 2)) logs = log.stop() + def test_closed_stderr(self): + """Test that first stage works with closed STDERR + + The boot command should read the preamble from STDIN, write all ECO + markers to STDOUT, and then execute the preamble. + + This test writes the complete preamble to blocking STDIN. + + 1. Fork child reads from blocking STDIN + 2. Fork child decompresses the data, does send the handshakes MITO001 and MITO002 + 3. Fork child crashes (when it tries to close the already closed + STDERR), but that's non-critical as the parent can read the data + already written by the fork child. + 4. Fork child's file descriptors (write pipes) are closed by the OS + 5. Fork parent does `dup(, )` and `exec()` + 6. Python reads all data from stdin + 7. Python runs the preamble code + 8. A context call works as expected. + + """ + log = testlib.LogCapturer() + log.start() + ctx = self.router._connect(DummyConnectionClosedStderr, connect_timeout=0.5) + self.assertEqual(3, ctx.call(operator.add, 1, 2)) + logs = log.stop() + class CommandLineTest(testlib.RouterMixin, testlib.TestCase): # Ensure this version of Python produces a command line that is sufficient From 2e4821b3aeda8645b420202814d47b8fa52504a5 Mon Sep 17 00:00:00 2001 From: Marc Hartmayer Date: Tue, 9 Dec 2025 19:37:21 +0000 Subject: [PATCH 13/13] NOT NECESSARY mitogen/parent: Ignore if STDERR is not available If STDERR is not available, ignore the OSError since it's a non-critical error. Note: This change is not necessary as the exception message would be print on stderr and stderr is already closed and the exit status of the forked child process is not checked yet. Signed-off-by: Marc Hartmayer --- mitogen/parent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 5349c6ef8..217a1ad75 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1465,7 +1465,10 @@ def _first_stage(): f.write(C) f.close() os.write(1,'MITO001\n'.encode()) - os.close(2) + try: + os.close(2) + except OSError: + pass def get_python_argv(self): """