Faster subprocesses by not using Process Pool

Author: rotu

colcon_python_setup_py/package_identification/out_of_process.py

@@ -0,0 +1,47 @@
+import functools
+import traceback
+
+
+class OutOfProcessError(RuntimeError):
+    """An exception raised from a different process"""
+
+    def __init__(self, description):
+        self.description = description
+
+    def __str__(self):
+        return self.description
+
+
+def out_of_process(fn):
+    """Decorator to wrap a function call in a subprocess"""
+
+    @functools.wraps(fn)
+    def wrapper(*args, **kwargs):
+        import multiprocessing
+
+        parent_conn, child_conn = multiprocessing.Pipe(duplex=False)
+
+        def target():
+            with child_conn:
+                try:
+                    result = fn(*args, **kwargs)
+                    child_conn.send((True, result))
+                except BaseException:
+                    child_conn.send((False, traceback.format_exc()))
+
+        with parent_conn:
+            p = multiprocessing.Process(
+                target=target,
+                daemon=True, name='out_of_process ' + fn.__name__
+            )
+            try:
+                p.start()
+                ok, value = parent_conn.recv()
+                if ok:
+                    return value
+                else:
+                    raise OutOfProcessError(value)
+            finally:
+                p.terminate()
+
+    return wrapper

colcon_python_setup_py/package_identification/python_setup_py.py

@@ -4,7 +4,6 @@
 
 import ast
 import distutils.core
-import multiprocessing
 import os
 from pathlib import Path
 import runpy
@@ -20,6 +19,7 @@
 from colcon_core.package_identification.python import \
     create_dependency_descriptor
 from colcon_core.plugin_system import satisfies_version
+from .out_of_process import out_of_process
 
 from .run_setup_py import run_setup_py
 
@@ -245,19 +245,16 @@ def get_setup_information(setup_py: Path, *, env: Mapping[str, str]):
     :return: dictionary of data describing the package.
     :raise: RuntimeError if the setup script encountered an error
     """
+    setup_in_subprocess = out_of_process(run_setup_py)
     try:
-        with multiprocessing.Pool(1) as _process_pool:
-            return _process_pool.apply(
-                run_setup_py,
-                kwds={
-                    'cwd': os.path.abspath(str(setup_py.parent)),
-                    # might be os.environ, which is not picklable
-                    'env': dict(env),
-                    'script_args': ('--dry-run',),
-                    'stop_after': 'config'
-                }
-            )
+        setup_in_subprocess(
+            cwd=os.path.abspath(str(setup_py.parent)),
+            # might be os.environ, which is not picklable
+            env=dict(env),
+            script_args=('--dry-run',),
+            stop_after='config'
+        )
     except Exception as e:
         raise RuntimeError(
             "Failed to dry run setup script '{setup_py}': "
-            .format_map(locals()) + traceback.format_exc()) from e
+                .format_map(locals()) + traceback.format_exc()) from e