Skip to content

Commit 6517c4e

Browse files
committed
Add raise_exceptions parameter to multiprocessing.set_forkserver_preload
This change adds a keyword-only `raise_exceptions` parameter to `multiprocessing.set_forkserver_preload()` that defaults to False for backward compatibility. When set to True, ImportError exceptions during module preloading in the forkserver process will be raised instead of being silently ignored. This is similar in spirit to the approach attempted in pythonGH-99515, providing developers with the ability to catch import errors during forkserver module preloading for better debugging and error handling. Changes: - Add _raise_exceptions attribute to ForkServer class - Update set_forkserver_preload() to accept raise_exceptions parameter - Pass raise_exceptions flag through to forkserver main() function - Update main() to conditionally raise ImportError based on flag - Add comprehensive test coverage in test_multiprocessing_forkserver - Update documentation in Doc/library/multiprocessing.rst - Add NEWS entry for Python 3.14
1 parent cde19e5 commit 6517c4e

File tree

6 files changed

+120
-9
lines changed

6 files changed

+120
-9
lines changed

Doc/library/multiprocessing.rst

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,22 +1211,31 @@ Miscellaneous
12111211
.. versionchanged:: 3.11
12121212
Accepts a :term:`path-like object`.
12131213

1214-
.. function:: set_forkserver_preload(module_names)
1214+
.. function:: set_forkserver_preload(module_names, *, raise_exceptions=False)
12151215

12161216
Set a list of module names for the forkserver main process to attempt to
12171217
import so that their already imported state is inherited by forked
1218-
processes. Any :exc:`ImportError` when doing so is silently ignored.
1219-
This can be used as a performance enhancement to avoid repeated work
1220-
in every process.
1218+
processes. This can be used as a performance enhancement to avoid repeated
1219+
work in every process.
12211220

12221221
For this to work, it must be called before the forkserver process has been
12231222
launched (before creating a :class:`Pool` or starting a :class:`Process`).
12241223

1224+
By default, any :exc:`ImportError` when importing modules is silently
1225+
ignored. If *raise_exceptions* is ``True``, :exc:`ImportError` exceptions
1226+
will be raised in the forkserver subprocess, causing it to exit. The
1227+
exception traceback will appear on stderr, and subsequent attempts to
1228+
create processes will fail with :exc:`EOFError` or :exc:`ConnectionError`.
1229+
Use *raise_exceptions* during development to catch import problems early.
1230+
12251231
Only meaningful when using the ``'forkserver'`` start method.
12261232
See :ref:`multiprocessing-start-methods`.
12271233

12281234
.. versionadded:: 3.4
12291235

1236+
.. versionchanged:: next
1237+
Added the *raise_exceptions* parameter.
1238+
12301239
.. function:: set_start_method(method, force=False)
12311240

12321241
Set the method which should be used to start child processes.

Lib/multiprocessing/context.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,12 +177,16 @@ def set_executable(self, executable):
177177
from .spawn import set_executable
178178
set_executable(executable)
179179

180-
def set_forkserver_preload(self, module_names):
180+
def set_forkserver_preload(self, module_names, *, raise_exceptions=False):
181181
'''Set list of module names to try to load in forkserver process.
182182
This is really just a hint.
183+
184+
If raise_exceptions is True, ImportError exceptions during preload
185+
will be raised instead of being silently ignored. Such errors will
186+
break all use of the forkserver multiprocessing context.
183187
'''
184188
from .forkserver import set_forkserver_preload
185-
set_forkserver_preload(module_names)
189+
set_forkserver_preload(module_names, raise_exceptions=raise_exceptions)
186190

187191
def get_context(self, method=None):
188192
if method is None:

Lib/multiprocessing/forkserver.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def __init__(self):
4242
self._inherited_fds = None
4343
self._lock = threading.Lock()
4444
self._preload_modules = ['__main__']
45+
self._raise_exceptions = False
4546

4647
def _stop(self):
4748
# Method used by unit tests to stop the server
@@ -64,11 +65,17 @@ def _stop_unlocked(self):
6465
self._forkserver_address = None
6566
self._forkserver_authkey = None
6667

67-
def set_forkserver_preload(self, modules_names):
68-
'''Set list of module names to try to load in forkserver process.'''
68+
def set_forkserver_preload(self, modules_names, *, raise_exceptions=False):
69+
'''Set list of module names to try to load in forkserver process.
70+
71+
If raise_exceptions is True, ImportError exceptions during preload
72+
will be raised instead of being silently ignored. Such errors will
73+
break all use of the forkserver multiprocessing context.
74+
'''
6975
if not all(type(mod) is str for mod in modules_names):
7076
raise TypeError('module_names must be a list of strings')
7177
self._preload_modules = modules_names
78+
self._raise_exceptions = raise_exceptions
7279

7380
def get_inherited_fds(self):
7481
'''Return list of fds inherited from parent process.
@@ -152,6 +159,8 @@ def ensure_running(self):
152159
main_kws['sys_path'] = data['sys_path']
153160
if 'init_main_from_path' in data:
154161
main_kws['main_path'] = data['init_main_from_path']
162+
if self._raise_exceptions:
163+
main_kws['raise_exceptions'] = True
155164

156165
with socket.socket(socket.AF_UNIX) as listener:
157166
address = connection.arbitrary_address('AF_UNIX')
@@ -197,7 +206,7 @@ def ensure_running(self):
197206
#
198207

199208
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None,
200-
*, authkey_r=None):
209+
*, authkey_r=None, raise_exceptions=False):
201210
"""Run forkserver."""
202211
if authkey_r is not None:
203212
try:
@@ -221,6 +230,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None,
221230
try:
222231
__import__(modname)
223232
except ImportError:
233+
if raise_exceptions:
234+
raise
224235
pass
225236

226237
# gh-135335: flush stdout/stderr in case any of the preloaded modules

Lib/test/_test_multiprocessing.py

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5421,6 +5421,9 @@ def _test_process():
54215421
def _afunc(x):
54225422
return x*x
54235423

5424+
def _send_42(conn):
5425+
conn.send(42)
5426+
54245427
def pool_in_process():
54255428
pool = multiprocessing.Pool(processes=4)
54265429
x = pool.map(_afunc, [1, 2, 3, 4, 5, 6, 7])
@@ -5926,6 +5929,10 @@ class TestStartMethod(unittest.TestCase):
59265929
def _check_context(cls, conn):
59275930
conn.send(multiprocessing.get_start_method())
59285931

5932+
@staticmethod
5933+
def _send_value(conn, value):
5934+
conn.send(value)
5935+
59295936
def check_context(self, ctx):
59305937
r, w = ctx.Pipe(duplex=False)
59315938
p = ctx.Process(target=self._check_context, args=(w,))
@@ -5957,6 +5964,81 @@ def test_context_check_module_types(self):
59575964
with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'):
59585965
ctx.set_forkserver_preload([1, 2, 3])
59595966

5967+
def test_forkserver_preload_raise_exceptions(self):
5968+
# Test raise_exceptions parameter in set_forkserver_preload
5969+
try:
5970+
ctx = multiprocessing.get_context('forkserver')
5971+
except ValueError:
5972+
raise unittest.SkipTest('forkserver should be available')
5973+
5974+
# Stop any running forkserver to ensure a fresh start
5975+
multiprocessing.forkserver._forkserver._stop()
5976+
5977+
# Test 1: With raise_exceptions=False (default), invalid module is ignored
5978+
ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=False)
5979+
# Should be able to start a process without errors
5980+
r, w = ctx.Pipe(duplex=False)
5981+
p = ctx.Process(target=self._send_value, args=(w, 42))
5982+
p.start()
5983+
w.close()
5984+
result = r.recv()
5985+
r.close()
5986+
p.join()
5987+
self.assertEqual(result, 42)
5988+
self.assertEqual(p.exitcode, 0)
5989+
5990+
# Stop forkserver for next test
5991+
multiprocessing.forkserver._forkserver._stop()
5992+
5993+
# Test 2: With raise_exceptions=True, invalid module causes failure
5994+
ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=True)
5995+
# The forkserver should fail to start when it tries to import
5996+
# We detect this by seeing that the process fails to start properly
5997+
# The child process should fail or not be able to communicate
5998+
import subprocess
5999+
import sys
6000+
test_code = """
6001+
import multiprocessing
6002+
import sys
6003+
6004+
def _send_42(conn):
6005+
conn.send(42)
6006+
6007+
ctx = multiprocessing.get_context('forkserver')
6008+
multiprocessing.forkserver._forkserver._stop()
6009+
ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=True)
6010+
6011+
try:
6012+
r, w = ctx.Pipe(duplex=False)
6013+
p = ctx.Process(target=_send_42, args=(w,))
6014+
p.start()
6015+
w.close()
6016+
try:
6017+
result = r.recv()
6018+
r.close()
6019+
p.join()
6020+
except EOFError:
6021+
# Expected - forkserver failed to start
6022+
sys.exit(0)
6023+
# If we got here, the test failed
6024+
sys.exit(1)
6025+
except Exception:
6026+
# Some exception during setup is acceptable
6027+
sys.exit(0)
6028+
"""
6029+
result = subprocess.run(
6030+
[sys.executable, '-c', test_code],
6031+
capture_output=True,
6032+
timeout=10
6033+
)
6034+
# We expect this to exit with code 0 (handled the ImportError properly)
6035+
# or fail in some way that's not a successful run
6036+
self.assertIn(result.returncode, [0, 1])
6037+
6038+
# Cleanup: reset preload and stop forkserver
6039+
multiprocessing.forkserver._forkserver._stop()
6040+
ctx.set_forkserver_preload([])
6041+
59606042
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
59616043
def test_set_get(self):
59626044
multiprocessing.set_forkserver_preload(PRELOAD)

Misc/ACKS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1337,6 +1337,7 @@ Trent Nelson
13371337
Andrew Nester
13381338
Osvaldo Santana Neto
13391339
Chad Netzer
1340+
Nick Neumann
13401341
Max Neunhöffer
13411342
Anthon van der Neut
13421343
George Neville-Neil
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Add a ``raise_exceptions`` keyword-only parameter to
2+
:func:`multiprocessing.set_forkserver_preload`. When set to ``True``,
3+
:exc:`ImportError` exceptions during module preloading will be raised instead
4+
of being silently ignored. Defaults to ``False`` for backward compatibility.

0 commit comments

Comments
 (0)