Make subprocess calls faster by not using ProcessPoolExecutor

colcon_python_setup_py/package_identification/out_of_process.py

@@ -0,0 +1,64 @@
+# Copyright 2019 Rover Robotics via Dan Rose
+# Licensed under the Apache License, Version 2.0
+
+import functools
+import traceback
+
+
+class OutOfProcessError(RuntimeError):
+    """An exception raised from a different process."""
+
+    def __init__(self, description):
+        """
+        Initialize self.
+
+        :param description: The full formatted exception
+        """
+        self.description = description
+
+    def __str__(self):
+        """Return str(self)."""
+        return self.description
+
+
+def out_of_process(fn):
+    """
+    Wrap a function in a subprocess.
+
+    Wrapped function will behave the same as the original function, except
+    it will run in a different process and exceptions will be wrapped in an
+    OutOfProcessError
+
+    :param fn: Function to run in a separate process
+    :return: Function that mimics `fn`.
+    """
+    @functools.wraps(fn)
+    def wrapper(*args, **kwargs):
+        import multiprocessing
+
+        parent_conn, child_conn = multiprocessing.Pipe(duplex=False)
+
+        def target():
+            with child_conn:
+                try:
+                    result = fn(*args, **kwargs)
+                    child_conn.send((True, result))
+                except BaseException:
+                    child_conn.send((False, traceback.format_exc()))
+
+        with parent_conn:
+            p = multiprocessing.Process(
+                target=target,
+                daemon=True, name='out_of_process ' + fn.__name__
+            )
+            try:
+                p.start()
+                ok, value = parent_conn.recv()
+                if ok:
+                    return value
+                else:
+                    raise OutOfProcessError(value)
+            finally:
+                p.terminate()
+
+    return wrapper

colcon_python_setup_py/package_identification/python_setup_py.py

@@ -1,25 +1,27 @@
-# Copyright 2016-2018 Dirk Thomas
+# Copyright 2016-2019 Dirk Thomas
+# Copyright 2019 Rover Robotics via Dan Rose
 # Licensed under the Apache License, Version 2.0
 
 import ast
 import distutils.core
 import os
 from pathlib import Path
 import runpy
-try:
-    import setuptools
-except ImportError:
-    pass
 import subprocess
 import sys
 from threading import Lock
+import traceback
+from typing import Mapping
+import warnings
 
-from colcon_core.package_identification import logger
 from colcon_core.package_identification \
     import PackageIdentificationExtensionPoint
 from colcon_core.package_identification.python import \
     create_dependency_descriptor
 from colcon_core.plugin_system import satisfies_version
+from .out_of_process import out_of_process
+
+from .run_setup_py import run_setup_py
 
 
 class PythonPackageIdentification(PackageIdentificationExtensionPoint):
@@ -43,31 +45,35 @@ def identify(self, desc):  # noqa: D102
         if not setup_py.is_file():
             return
 
-        kwargs = get_setup_arguments(setup_py)
-        data = extract_data(**kwargs)
+        config = get_setup_information(setup_py, env=os.environ)
+
+        name = config['metadata'].name
+        if not name:
+            raise RuntimeError(
+                'Failed to determine Python package name in '
+                "'{setup_py.parent}'".format_map(locals()))
 
-        if desc.type is not None and desc.type != 'python':
-            logger.error('Package type already set to different value')
-            raise RuntimeError('Package type already set to different value')
         desc.type = 'python'
-        if desc.name is not None and desc.name != data['name']:
-            logger.error('Package name already set to different value')
-            raise RuntimeError('Package name already set to different value')
-        desc.name = data['name']
-        for key in ('build', 'run', 'test'):
-            desc.dependencies[key] |= data['%s_depends' % key]
+        if desc.name is None:
+            desc.name = name
+
+        desc.metadata['version'] = config['metadata'].version
 
-        path = str(desc.path)
+        for dependency_type, option_name in [
+            ('build', 'setup_requires'),
+            ('run', 'install_requires'),
+            ('test', 'tests_require')
+        ]:
+            desc.dependencies[dependency_type] = {
+                create_dependency_descriptor(d)
+                for d in config[option_name] or ()}
 
         def getter(env):
-            nonlocal path
-            return get_setup_arguments_with_context(
-                os.path.join(path, 'setup.py'), env)
+            nonlocal setup_py
+            return get_setup_information(setup_py, env=env)
 
         desc.metadata['get_python_setup_options'] = getter
 
-        desc.metadata['version'] = getter(os.environ)['version']
-
 
 cwd_lock = None
 
@@ -84,6 +90,13 @@ def get_setup_arguments(setup_py):
     :returns: a dictionary containing the arguments of the setup() function
     :rtype: dict
     """
+    warnings.warn(
+        'colcon_python_setup_py.package_identification.python_setup_py.'
+        'get_setup_arguments() has been deprecated, use '
+        'colcon_python_setup_py.package_identification.python_setup_py.'
+        'get_setup_information() instead',
+        stacklevel=2)
+    import setuptools
     global cwd_lock
     if not cwd_lock:
         cwd_lock = Lock()
@@ -137,6 +150,11 @@ def create_mock_setup_function(data):
     :returns: a function to replace distutils.core.setup and setuptools.setup
     :rtype: callable
     """
+    warnings.warn(
+        'colcon_python_setup_py.package_identification.python_setup_py.'
+        'create_mock_setup_function() will be removed in the future',
+        DeprecationWarning, stacklevel=2)
+
     def setup(*args, **kwargs):
         if args:
             raise RuntimeError(
@@ -160,6 +178,10 @@ def extract_data(**kwargs):
     :rtype: dict
     :raises RuntimeError: if the keywords don't contain `name`
     """
+    warnings.warn(
+        'colcon_python_setup_py.package_identification.python_setup_py.'
+        'extract_data() will be removed in the future',
+        DeprecationWarning, stacklevel=2)
     if 'name' not in kwargs:
         raise RuntimeError(
             "setup() function invoked without the keyword argument 'name'")
@@ -186,6 +208,8 @@ def get_setup_arguments_with_context(setup_py, env):
     a separate Python interpreter is being used which can have an extended
     PYTHONPATH etc.
 
+    This function has been deprecated, use get_setup_information() instead.
+
     :param setup_py: The path of the setup.py file
     :param dict env: The environment variables to use when invoking the file
     :returns: a dictionary containing the arguments of the setup() function
@@ -210,3 +234,27 @@ def get_setup_arguments_with_context(setup_py, env):
     output = result.stdout.decode('utf-8')
 
     return ast.literal_eval(output)
+
+
+def get_setup_information(setup_py: Path, *, env: Mapping[str, str]):
+    """
+    Dry run the setup.py file and get the configuration information.
+
+    :param setup_py: path to a setup.py script
+    :param env: environment variables to set before running setup.py
+    :return: dictionary of data describing the package.
+    :raise: RuntimeError if the setup script encountered an error
+    """
+    setup_in_subprocess = out_of_process(run_setup_py)
+    try:
+        setup_in_subprocess(
+            cwd=os.path.abspath(str(setup_py.parent)),
+            # might be os.environ, which is not picklable
+            env=dict(env),
+            script_args=('--dry-run',),
+            stop_after='config'
+        )
+    except Exception as e:
+        raise RuntimeError(
+            "Failed to dry run setup script '{setup_py}': "
+            .format_map(locals()) + traceback.format_exc()) from e

colcon_python_setup_py/package_identification/run_setup_py.py

@@ -0,0 +1,51 @@
+# Copyright 2019 Rover Robotics via Dan Rose
+# Licensed under the Apache License, Version 2.0
+
+import distutils.core
+import os
+
+
+def run_setup_py(cwd, env, script_args=(), stop_after='run'):
+    """
+    Modify the current process and run setup.py.
+
+    This should be run in a subprocess to not affect the state of the current
+    process.
+
+    :param str cwd: absolute path to a directory containing a setup.py script
+    :param dict env: environment variables to set before running setup.py
+    :param script_args: command-line arguments to pass to setup.py
+    :param stop_after: tells setup() when to stop processing
+    :returns: the public properties of a Distribution object, minus objects
+      with are generally not picklable
+    """
+    # need to be in setup.py's parent dir to detect any setup.cfg
+    os.chdir(cwd)
+
+    os.environ.clear()
+    os.environ.update(env)
+
+    result = distutils.core.run_setup(
+        'setup.py', script_args=script_args, stop_after=stop_after)
+
+    try:
+        # hack to make this class pickle to an ordinary set
+        from setuptools.extern.ordered_set import OrderedSet
+        OrderedSet.__reduce__ = lambda self: (set, (list(self),))
+    except ImportError:
+        pass
+
+    return {
+        key: value for key, value in result.__dict__.items()
+        if (
+            # Private properties
+            not key.startswith('_') and
+            # Getter methods
+            not callable(value) and
+            # Objects that are generally not picklable
+            key not in ('cmdclass', 'distclass', 'ext_modules') and
+            # These *seem* useful but always have the value 0.
+            # Look for their values in the 'metadata' object instead.
+            key not in result.display_option_names
+        )
+    }

test/spell_check.words

@@ -1,14 +1,20 @@
 apache
 chdir
 colcon
+distclass
+functools
 iterdir
 noqa
 pathlib
+picklable
 plugin
 pytest
 pythonpath
+recv
 rtype
 runpy
 scspell
 setuptools
+stacklevel
 thomas
+traceback