diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a5b52e..b6ae06b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Packaging - Drop support for Python 3.8. [#479](https://github.com/PyO3/setuptools-rust/pull/479) - Support free-threaded Python. [#502](https://github.com/PyO3/setuptools-rust/pull/502) +- Support adding custom env vars. [#504](https://github.com/PyO3/setuptools-rust/pull/504) ## 1.10.2 (2024-10-02) ### Fixed diff --git a/emscripten/.ruff.toml b/emscripten/.ruff.toml new file mode 100644 index 00000000..abf8eeea --- /dev/null +++ b/emscripten/.ruff.toml @@ -0,0 +1,2 @@ +[lint] +extend-ignore = ["TID251"] diff --git a/noxfile.py b/noxfile.py index 3565dbff..cc93b46b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -139,7 +139,7 @@ def test_crossenv(session: nox.Session): @nox.session() def ruff(session: nox.Session): session.install("ruff") - session.run("ruff", "format", "--check", ".") + session.run("ruff", "format", "--diff", ".") session.run("ruff", "check", ".") diff --git a/pyproject.toml b/pyproject.toml index 09c2409c..1f08d6d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,16 @@ Changelog = "https://github.com/PyO3/setuptools-rust/blob/main/CHANGELOG.md" requires = ["setuptools>=62.4", "setuptools_scm"] build-backend = "setuptools.build_meta" +[tool.ruff.lint] +extend-select = ["TID251"] + [tool.ruff.lint.extend-per-file-ignores] "__init__.py" = ["F403"] +[tool.ruff.lint.flake8-tidy-imports.banned-api] +"subprocess.run".msg = "Use `_utils.run_subprocess` to ensure `env` is passed" +"subprocess.check_output".msg = "Use `_utils.check_subprocess_output` to ensure `env` is passed" + [tool.pytest.ini_options] minversion = "6.0" addopts = "--doctest-modules" diff --git a/setuptools_rust/_utils.py b/setuptools_rust/_utils.py index b6d9400b..62c7d72a 100644 --- a/setuptools_rust/_utils.py +++ b/setuptools_rust/_utils.py @@ -1,4 +1,48 @@ import subprocess +from typing import Any, Optional, Union, cast + + +class Env: + """Allow using ``functools.lru_cache`` with an environment variable dictionary. + + Dictionaries are unhashable, but ``functools.lru_cache`` needs all parameters to + be hashable, which we solve which a custom ``__hash__``.""" + + env: Optional[dict[str, str]] + + def __init__(self, env: Optional[dict[str, str]]): + self.env = env + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Env): + return False + return self.env == other.env + + def __hash__(self) -> int: + if self.env is not None: + return hash(tuple(sorted(self.env.items()))) + else: + return hash(None) + + +def run_subprocess( + *args: Any, env: Union[Env, dict[str, str], None], **kwargs: Any +) -> subprocess.CompletedProcess: + """Wrapper around subprocess.run that requires a decision to pass env.""" + if isinstance(env, Env): + env = env.env + kwargs["env"] = env + return subprocess.run(*args, **kwargs) # noqa: TID251 # this is a wrapper to implement the rule + + +def check_subprocess_output( + *args: Any, env: Union[Env, dict[str, str], None], **kwargs: Any +) -> str: + """Wrapper around subprocess.run that requires a decision to pass env.""" + if isinstance(env, Env): + env = env.env + kwargs["env"] = env + return cast(str, subprocess.check_output(*args, **kwargs)) # noqa: TID251 # this is a wrapper to implement the rule def format_called_process_error( diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index feb93198..98b20e41 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -24,7 +24,7 @@ from setuptools.command.build_ext import get_abi3_suffix from setuptools.command.install_scripts import install_scripts as CommandInstallScripts -from ._utils import format_called_process_error +from ._utils import check_subprocess_output, format_called_process_error, Env from .command import RustCommand from .extension import Binding, RustBin, RustExtension, Strip from .rustc_info import ( @@ -45,8 +45,8 @@ from setuptools import Command as CommandBdistWheel # type: ignore[assignment] -def _check_cargo_supports_crate_type_option() -> bool: - version = get_rust_version() +def _check_cargo_supports_crate_type_option(env: Optional[Env]) -> bool: + version = get_rust_version(env) if version is None: return False @@ -144,10 +144,10 @@ def run_for_extension(self, ext: RustExtension) -> None: def build_extension( self, ext: RustExtension, forced_target_triple: Optional[str] = None ) -> List["_BuiltModule"]: - target_triple = self._detect_rust_target(forced_target_triple) - rustc_cfgs = get_rustc_cfgs(target_triple) + target_triple = self._detect_rust_target(forced_target_triple, ext.env) + rustc_cfgs = get_rustc_cfgs(target_triple, ext.env) - env = _prepare_build_environment() + env = _prepare_build_environment(ext.env) if not os.path.exists(ext.path): raise FileError( @@ -156,7 +156,7 @@ def build_extension( quiet = self.qbuild or ext.quiet debug = self._is_debug_build(ext) - use_cargo_crate_type = _check_cargo_supports_crate_type_option() + use_cargo_crate_type = _check_cargo_supports_crate_type_option(ext.env) package_id = ext.metadata(quiet=quiet)["resolve"]["root"] if package_id is None: @@ -252,7 +252,7 @@ def build_extension( # If quiet, capture all output and only show it in the exception # If not quiet, forward all cargo output to stderr stderr = subprocess.PIPE if quiet else None - cargo_messages = subprocess.check_output( + cargo_messages = check_subprocess_output( command, env=env, stderr=stderr, @@ -417,7 +417,7 @@ def install_extension( args.insert(0, "strip") args.append(ext_path) try: - subprocess.check_output(args) + check_subprocess_output(args, env=None) except subprocess.CalledProcessError: pass @@ -477,7 +477,7 @@ def _py_limited_api(self) -> _PyLimitedApi: return cast(_PyLimitedApi, bdist_wheel.py_limited_api) def _detect_rust_target( - self, forced_target_triple: Optional[str] = None + self, forced_target_triple: Optional[str], env: Env ) -> Optional[str]: assert self.plat_name is not None if forced_target_triple is not None: @@ -486,14 +486,14 @@ def _detect_rust_target( return forced_target_triple # Determine local rust target which needs to be "forced" if necessary - local_rust_target = _adjusted_local_rust_target(self.plat_name) + local_rust_target = _adjusted_local_rust_target(self.plat_name, env) # Match cargo's behaviour of not using an explicit target if the # target we're compiling for is the host if ( local_rust_target is not None # check for None first to avoid calling to rustc if not needed - and local_rust_target != get_rust_host() + and local_rust_target != get_rust_host(env) ): return local_rust_target @@ -566,7 +566,7 @@ def create_universal2_binary(output_path: str, input_paths: List[str]) -> None: # Try lipo first command = ["lipo", "-create", "-output", output_path, *input_paths] try: - subprocess.check_output(command, text=True) + check_subprocess_output(command, env=None, text=True) except subprocess.CalledProcessError as e: output = e.output raise CompileError("lipo failed with code: %d\n%s" % (e.returncode, output)) @@ -609,7 +609,7 @@ def _replace_vendor_with_unknown(target: str) -> Optional[str]: return "-".join(components) -def _prepare_build_environment() -> Dict[str, str]: +def _prepare_build_environment(env: Env) -> Dict[str, str]: """Prepares environment variables to use when executing cargo build.""" base_executable = None @@ -625,20 +625,18 @@ def _prepare_build_environment() -> Dict[str, str]: # executing python interpreter. bindir = os.path.dirname(executable) - env = os.environ.copy() - env.update( + env_vars = (env.env or os.environ).copy() + env_vars.update( { # disables rust's pkg-config seeking for specified packages, # which causes pythonXX-sys to fall back to detecting the # interpreter from the path. - "PATH": os.path.join(bindir, os.environ.get("PATH", "")), - "PYTHON_SYS_EXECUTABLE": os.environ.get( - "PYTHON_SYS_EXECUTABLE", executable - ), - "PYO3_PYTHON": os.environ.get("PYO3_PYTHON", executable), + "PATH": os.path.join(bindir, env_vars.get("PATH", "")), + "PYTHON_SYS_EXECUTABLE": env_vars.get("PYTHON_SYS_EXECUTABLE", executable), + "PYO3_PYTHON": env_vars.get("PYO3_PYTHON", executable), } ) - return env + return env_vars def _is_py_limited_api( @@ -692,19 +690,19 @@ def _binding_features( _PyLimitedApi = Literal["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", True, False] -def _adjusted_local_rust_target(plat_name: str) -> Optional[str]: +def _adjusted_local_rust_target(plat_name: str, env: Env) -> Optional[str]: """Returns the local rust target for the given `plat_name`, if it is necessary to 'force' a specific target for correctness.""" # If we are on a 64-bit machine, but running a 32-bit Python, then # we'll target a 32-bit Rust build. if plat_name == "win32": - if get_rustc_cfgs(None).get("target_env") == "gnu": + if get_rustc_cfgs(None, env).get("target_env") == "gnu": return "i686-pc-windows-gnu" else: return "i686-pc-windows-msvc" elif plat_name == "win-amd64": - if get_rustc_cfgs(None).get("target_env") == "gnu": + if get_rustc_cfgs(None, env).get("target_env") == "gnu": return "x86_64-pc-windows-gnu" else: return "x86_64-pc-windows-msvc" diff --git a/setuptools_rust/clean.py b/setuptools_rust/clean.py index b8e9ed3a..e1a132c2 100644 --- a/setuptools_rust/clean.py +++ b/setuptools_rust/clean.py @@ -1,6 +1,7 @@ -import subprocess import sys +from setuptools_rust._utils import check_subprocess_output + from .command import RustCommand from .extension import RustExtension @@ -25,6 +26,6 @@ def run_for_extension(self, ext: RustExtension) -> None: # Execute cargo command try: - subprocess.check_output(args) + check_subprocess_output(args, env=ext.env) except Exception: pass diff --git a/setuptools_rust/command.py b/setuptools_rust/command.py index e9f5c2be..1aa5c08d 100644 --- a/setuptools_rust/command.py +++ b/setuptools_rust/command.py @@ -52,8 +52,16 @@ def run(self) -> None: return all_optional = all(ext.optional for ext in self.extensions) + # Use the environment of the first non-optional extension, or the first optional + # extension if there is no non-optional extension. + env = None + for ext in self.extensions: + if ext.env: + env = ext.env + if not ext.optional: + break try: - version = get_rust_version() + version = get_rust_version(env) if version is None: min_version = max( # type: ignore[type-var] filter( diff --git a/setuptools_rust/extension.py b/setuptools_rust/extension.py index e305960e..65e8b067 100644 --- a/setuptools_rust/extension.py +++ b/setuptools_rust/extension.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from semantic_version import SimpleSpec -from ._utils import format_called_process_error +from ._utils import check_subprocess_output, format_called_process_error, Env class Binding(IntEnum): @@ -112,6 +112,9 @@ class RustExtension: abort the build process, and instead simply not install the failing extension. py_limited_api: Deprecated. + env: Environment variables to use when calling cargo or rustc (``env=`` + in ``subprocess.Popen``). setuptools-rust may add additional + variables or modify ``PATH``. """ def __init__( @@ -131,6 +134,7 @@ def __init__( native: bool = False, optional: bool = False, py_limited_api: Literal["auto", True, False] = "auto", + env: Optional[Dict[str, str]] = None, ): if isinstance(target, dict): name = "; ".join("%s=%s" % (key, val) for key, val in target.items()) @@ -153,6 +157,7 @@ def __init__( self.script = script self.optional = optional self.py_limited_api = py_limited_api + self.env = Env(env) if native: warnings.warn( @@ -260,8 +265,8 @@ def _metadata(self, cargo: str, quiet: bool) -> "CargoMetadata": # If quiet, capture stderr and only show it on exceptions # If not quiet, let stderr be inherited stderr = subprocess.PIPE if quiet else None - payload = subprocess.check_output( - metadata_command, stderr=stderr, encoding="latin-1" + payload = check_subprocess_output( + metadata_command, stderr=stderr, encoding="latin-1", env=self.env.env ) except subprocess.CalledProcessError as e: raise SetupError(format_called_process_error(e)) @@ -319,6 +324,7 @@ def __init__( debug: Optional[bool] = None, strip: Strip = Strip.No, optional: bool = False, + env: Optional[dict[str, str]] = None, ): super().__init__( target=target, @@ -333,6 +339,7 @@ def __init__( optional=optional, strip=strip, py_limited_api=False, + env=env, ) def entry_points(self) -> List[str]: diff --git a/setuptools_rust/rustc_info.py b/setuptools_rust/rustc_info.py index 58dfd42a..a46f34a7 100644 --- a/setuptools_rust/rustc_info.py +++ b/setuptools_rust/rustc_info.py @@ -5,17 +5,19 @@ from functools import lru_cache from typing import Dict, List, NewType, Optional, TYPE_CHECKING +from ._utils import Env, check_subprocess_output + if TYPE_CHECKING: from semantic_version import Version -def get_rust_version() -> Optional[Version]: # type: ignore[no-any-unimported] +def get_rust_version(env: Optional[Env]) -> Optional[Version]: # type: ignore[no-any-unimported] try: # first line of rustc -Vv is something like # rustc 1.61.0 (fe5b13d68 2022-05-18) from semantic_version import Version - return Version(_rust_version().split(" ")[1]) + return Version(_rust_version(env).split(" ")[1]) except (subprocess.CalledProcessError, OSError): return None @@ -23,11 +25,11 @@ def get_rust_version() -> Optional[Version]: # type: ignore[no-any-unimported] _HOST_LINE_START = "host: " -def get_rust_host() -> str: +def get_rust_host(env: Optional[Env]) -> str: # rustc -Vv has a line denoting the host which cargo uses to decide the # default target, e.g. # host: aarch64-apple-darwin - for line in _rust_version_verbose().splitlines(): + for line in _rust_version_verbose(env).splitlines(): if line.startswith(_HOST_LINE_START): return line[len(_HOST_LINE_START) :].strip() raise PlatformError("Could not determine rust host") @@ -36,9 +38,9 @@ def get_rust_host() -> str: RustCfgs = NewType("RustCfgs", Dict[str, Optional[str]]) -def get_rustc_cfgs(target_triple: Optional[str]) -> RustCfgs: +def get_rustc_cfgs(target_triple: Optional[str], env: Env) -> RustCfgs: cfgs = RustCfgs({}) - for entry in get_rust_target_info(target_triple): + for entry in get_rust_target_info(target_triple, env): maybe_split = entry.split("=", maxsplit=1) if len(maybe_split) == 2: cfgs[maybe_split[0]] = maybe_split[1].strip('"') @@ -49,25 +51,27 @@ def get_rustc_cfgs(target_triple: Optional[str]) -> RustCfgs: @lru_cache() -def get_rust_target_info(target_triple: Optional[str] = None) -> List[str]: +def get_rust_target_info(target_triple: Optional[str], env: Env) -> List[str]: cmd = ["rustc", "--print", "cfg"] if target_triple: cmd.extend(["--target", target_triple]) - output = subprocess.check_output(cmd, text=True) + output = check_subprocess_output(cmd, env=env, text=True) return output.splitlines() @lru_cache() -def get_rust_target_list() -> List[str]: - output = subprocess.check_output(["rustc", "--print", "target-list"], text=True) +def get_rust_target_list(env: Env) -> List[str]: + output = check_subprocess_output( + ["rustc", "--print", "target-list"], env=env, text=True + ) return output.splitlines() @lru_cache() -def _rust_version() -> str: - return subprocess.check_output(["rustc", "-V"], text=True) +def _rust_version(env: Env) -> str: + return check_subprocess_output(["rustc", "-V"], env=env, text=True) @lru_cache() -def _rust_version_verbose() -> str: - return subprocess.check_output(["rustc", "-Vv"], text=True) +def _rust_version_verbose(env: Env) -> str: + return check_subprocess_output(["rustc", "-Vv"], env=env, text=True) diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index ce457742..eb6219a2 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -1,5 +1,4 @@ import os -import subprocess import sys import sysconfig import logging @@ -15,6 +14,7 @@ from setuptools.command.sdist import sdist from setuptools.dist import Distribution +from ._utils import Env, run_subprocess from .build import _get_bdist_wheel_cmd from .extension import Binding, RustBin, RustExtension, Strip @@ -96,7 +96,20 @@ def make_distribution(self) -> None: # # https://doc.rust-lang.org/cargo/commands/cargo-build.html#manifest-options cargo_manifest_args: Set[str] = set() + env: Optional[Env] = None + env_source: Optional[str] = None for ext in self.distribution.rust_extensions: + if env is not None: + if ext.env != env: + raise ValueError( + f"For vendoring, all extensions must have the same environment variables, " + f"but {env_source} and {ext.name} differ:\n" + f"{env_source}: {env}\n" + f"{ext.name}: {ext.env}" + ) + else: + env = ext.env + env_source = ext.name manifest_paths.append(ext.path) if ext.cargo_manifest_args: cargo_manifest_args.update(ext.cargo_manifest_args) @@ -120,7 +133,7 @@ def make_distribution(self) -> None: # set --manifest-path before vendor_path and after --sync to workaround that # See https://docs.rs/clap/latest/clap/struct.Arg.html#method.multiple for detail command.extend(["--manifest-path", manifest_paths[0], vendor_path]) - subprocess.run(command, check=True) + run_subprocess(command, env=env, check=True) cargo_config = _CARGO_VENDOR_CONFIG diff --git a/tests/test_build.py b/tests/test_build.py index 102433d2..eda3b6b6 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -5,20 +5,24 @@ def test_adjusted_local_rust_target_windows_msvc(): with mock.patch( - "setuptools_rust.rustc_info.get_rust_target_info", lambda _: ["target_env=msvc"] + "setuptools_rust.rustc_info.get_rust_target_info", + lambda _plat_name, _env: ["target_env=msvc"], ): - assert _adjusted_local_rust_target("win32") == "i686-pc-windows-msvc" - assert _adjusted_local_rust_target("win-amd64") == "x86_64-pc-windows-msvc" + assert _adjusted_local_rust_target("win32", None) == "i686-pc-windows-msvc" + assert ( + _adjusted_local_rust_target("win-amd64", None) == "x86_64-pc-windows-msvc" + ) def test_adjusted_local_rust_target_windows_gnu(): with mock.patch( - "setuptools_rust.rustc_info.get_rust_target_info", lambda _: ["target_env=gnu"] + "setuptools_rust.rustc_info.get_rust_target_info", + lambda _plat_name, _env: ["target_env=gnu"], ): - assert _adjusted_local_rust_target("win32") == "i686-pc-windows-gnu" - assert _adjusted_local_rust_target("win-amd64") == "x86_64-pc-windows-gnu" + assert _adjusted_local_rust_target("win32", None) == "i686-pc-windows-gnu" + assert _adjusted_local_rust_target("win-amd64", None) == "x86_64-pc-windows-gnu" def test_adjusted_local_rust_target_macos(): with mock.patch("platform.machine", lambda: "x86_64"): - assert _adjusted_local_rust_target("macosx-") == "x86_64-apple-darwin" + assert _adjusted_local_rust_target("macosx-", None) == "x86_64-apple-darwin"