From c24e6ddf392cb13c1ff64d5c57d0ba59d929a752 Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Wed, 23 Oct 2019 16:26:52 -0500 Subject: [PATCH 1/4] 1. Move dependency on setuptools closer to where it's needed 2. Scope multiprocessing Pool so a pool left open doesn't hang the process 3. Move run_setup_py to its own file so the subprocess doesn't need to import much --- .../package_identification/python_setup_py.py | 69 ++++--------------- .../package_identification/run_setup_py.py | 41 +++++++++++ 2 files changed, 54 insertions(+), 56 deletions(-) create mode 100644 colcon_python_setup_py/package_identification/run_setup_py.py diff --git a/colcon_python_setup_py/package_identification/python_setup_py.py b/colcon_python_setup_py/package_identification/python_setup_py.py index a1b61c3..11d74b9 100644 --- a/colcon_python_setup_py/package_identification/python_setup_py.py +++ b/colcon_python_setup_py/package_identification/python_setup_py.py @@ -8,10 +8,6 @@ import os from pathlib import Path import runpy -try: - import setuptools -except ImportError: - pass import subprocess import sys from threading import Lock @@ -25,6 +21,7 @@ create_dependency_descriptor from colcon_core.plugin_system import satisfies_version +from .run_setup_py import run_setup_py class PythonPackageIdentification(PackageIdentificationExtensionPoint): """Identify Python packages with `setup.py` files.""" @@ -98,6 +95,7 @@ def get_setup_arguments(setup_py): 'colcon_python_setup_py.package_identification.python_setup_py.' 'get_setup_information() instead', stacklevel=2) + import setuptools global cwd_lock if not cwd_lock: cwd_lock = Lock() @@ -237,9 +235,6 @@ def get_setup_arguments_with_context(setup_py, env): return ast.literal_eval(output) -_process_pool = multiprocessing.Pool() - - def get_setup_information(setup_py: Path, *, env: Mapping[str, str]): """ Dry run the setup.py file and get the configuration information. @@ -250,56 +245,18 @@ def get_setup_information(setup_py: Path, *, env: Mapping[str, str]): :raise: RuntimeError if the setup script encountered an error """ try: - return _process_pool.apply( - run_setup_py, - kwds={ - 'cwd': os.path.abspath(str(setup_py.parent)), - # might be os.environ, which is not picklable - 'env': dict(env), - 'script_args': ('--dry-run',), - 'stop_after': 'config' - } - ) + with multiprocessing.Pool(1) as _process_pool: + return _process_pool.apply( + run_setup_py, + kwds={ + 'cwd': os.path.abspath(str(setup_py.parent)), + # might be os.environ, which is not picklable + 'env': dict(env), + 'script_args': ('--dry-run',), + 'stop_after': 'config' + } + ) except Exception as e: raise RuntimeError( "Failed to dry run setup script '{setup_py}': " .format_map(locals()) + traceback.format_exc()) from e - - -def run_setup_py(cwd, env, script_args=(), stop_after='run'): - """ - Modify the current process and run setup.py. - - This should be run in a subprocess to not affect the state of the current - process. - - :param str cwd: absolute path to a directory containing a setup.py script - :param dict env: environment variables to set before running setup.py - :param script_args: command-line arguments to pass to setup.py - :param stop_after: tells setup() when to stop processing - :returns: the public properties of a Distribution object, minus objects - with are generally not picklable - """ - # need to be in setup.py's parent dir to detect any setup.cfg - os.chdir(cwd) - - os.environ.clear() - os.environ.update(env) - - result = distutils.core.run_setup( - 'setup.py', script_args=script_args, stop_after=stop_after) - - return { - key: value for key, value in result.__dict__.items() - if ( - # Private properties - not key.startswith('_') and - # Getter methods - not callable(value) and - # Objects that are generally not picklable - key not in ('cmdclass', 'distclass', 'ext_modules') and - # These *seem* useful but always have the value 0. - # Look for their values in the 'metadata' object instead. - key not in result.display_option_names - ) - } diff --git a/colcon_python_setup_py/package_identification/run_setup_py.py b/colcon_python_setup_py/package_identification/run_setup_py.py new file mode 100644 index 0000000..6e93589 --- /dev/null +++ b/colcon_python_setup_py/package_identification/run_setup_py.py @@ -0,0 +1,41 @@ +import distutils.core +import os + + +def run_setup_py(cwd, env, script_args=(), stop_after='run'): + """ + Modify the current process and run setup.py. + + This should be run in a subprocess to not affect the state of the current + process. + + :param str cwd: absolute path to a directory containing a setup.py script + :param dict env: environment variables to set before running setup.py + :param script_args: command-line arguments to pass to setup.py + :param stop_after: tells setup() when to stop processing + :returns: the public properties of a Distribution object, minus objects + with are generally not picklable + """ + # need to be in setup.py's parent dir to detect any setup.cfg + os.chdir(cwd) + + os.environ.clear() + os.environ.update(env) + + result = distutils.core.run_setup( + 'setup.py', script_args=script_args, stop_after=stop_after) + + return { + key: value for key, value in result.__dict__.items() + if ( + # Private properties + not key.startswith('_') and + # Getter methods + not callable(value) and + # Objects that are generally not picklable + key not in ('cmdclass', 'distclass', 'ext_modules') and + # These *seem* useful but always have the value 0. + # Look for their values in the 'metadata' object instead. + key not in result.display_option_names + ) + } From 339ab9b0853e75ce06760bb1f3bd702f50a37664 Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Thu, 24 Oct 2019 13:28:51 -0700 Subject: [PATCH 2/4] fix linter --- colcon_python_setup_py/package_identification/python_setup_py.py | 1 + 1 file changed, 1 insertion(+) diff --git a/colcon_python_setup_py/package_identification/python_setup_py.py b/colcon_python_setup_py/package_identification/python_setup_py.py index 11d74b9..188b8be 100644 --- a/colcon_python_setup_py/package_identification/python_setup_py.py +++ b/colcon_python_setup_py/package_identification/python_setup_py.py @@ -23,6 +23,7 @@ from .run_setup_py import run_setup_py + class PythonPackageIdentification(PackageIdentificationExtensionPoint): """Identify Python packages with `setup.py` files.""" From 6d2b4fc86e12c5950e870f0f60e4d2ab6715e91f Mon Sep 17 00:00:00 2001 From: Dirk Thomas Date: Thu, 24 Oct 2019 13:30:29 -0700 Subject: [PATCH 3/4] add copyright / license comment --- colcon_python_setup_py/package_identification/run_setup_py.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/colcon_python_setup_py/package_identification/run_setup_py.py b/colcon_python_setup_py/package_identification/run_setup_py.py index 6e93589..3523bc4 100644 --- a/colcon_python_setup_py/package_identification/run_setup_py.py +++ b/colcon_python_setup_py/package_identification/run_setup_py.py @@ -1,3 +1,6 @@ +# Copyright 2019 Rover Robotics via Dan Rose +# Licensed under the Apache License, Version 2.0 + import distutils.core import os From d8d25a0cd9b5aa9406c610c671fa630bf12ae2c5 Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Fri, 25 Oct 2019 02:00:33 -0500 Subject: [PATCH 4/4] Fix Setuptools Pickle Problem see https://github.com/pypa/setuptools/issues/1888 --- .../package_identification/run_setup_py.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/colcon_python_setup_py/package_identification/run_setup_py.py b/colcon_python_setup_py/package_identification/run_setup_py.py index 3523bc4..7368234 100644 --- a/colcon_python_setup_py/package_identification/run_setup_py.py +++ b/colcon_python_setup_py/package_identification/run_setup_py.py @@ -28,6 +28,13 @@ def run_setup_py(cwd, env, script_args=(), stop_after='run'): result = distutils.core.run_setup( 'setup.py', script_args=script_args, stop_after=stop_after) + try: + # hack to make this class pickle to an ordinary set + from setuptools.extern.ordered_set import OrderedSet + OrderedSet.__reduce__ = lambda self: (set, (list(self),)) + except ImportError: + pass + return { key: value for key, value in result.__dict__.items() if (