diff --git a/docs/notes/2.32.x.md b/docs/notes/2.32.x.md index 52dd804babe..22dde34ebc8 100644 --- a/docs/notes/2.32.x.md +++ b/docs/notes/2.32.x.md @@ -106,6 +106,14 @@ Copying the `mypy` cache back to the ["named cache"](https://www.pantsbuild.org/ The mypy subsystem now supports a new `cache_mode="none"` to disable mypy's caching entirely. This is slower and intended as an "escape valve". It is hoped that on the latest mypy with the above Pants side fixes it will not be necessary. +A new **experimental** `[python].pex_builder` option allows using [uv](https://github.com/astral-sh/uv) to install +dependencies when building PEX binaries via `pants package`. When set to `"uv"`, Pants creates a virtual environment +with uv, then passes it to PEX via `--venv-repository` so PEX packages from the pre-populated venv instead of +resolving with pip. When a PEX-native lockfile is available, uv installs the exact pinned versions from the lockfile +with `--no-deps`, preserving reproducibility. This only applies to non-internal, non-cross-platform PEX builds with +explicit requirement strings and a local Python interpreter; other builds silently fall back to pip. +See [#20679](https://github.com/pantsbuild/pants/issues/20679) for background. + The `runtime` field of [`aws_python_lambda_layer`](https://www.pantsbuild.org/2.32/reference/targets/python_aws_lambda_layer#runtime) or [`aws_python_lambda_function`](https://www.pantsbuild.org/2.32/reference/targets/python_aws_lambda_function#runtime) now has built-in complete platform configurations for x86-64 and arm64 Python 3.14. This provides stable support for Python 3.14 lambdas out of the box, allowing deleting manual `complete_platforms` configuration if any. The `grpc-python-plugin` tool now uses an updated `v1.73.1` plugin built from bool: ), advanced=True, ) + pex_builder = EnumOption( + default=PexBuilder.pex, + help=softwrap( + """ + Which tool to use for installing dependencies when building PEX files. + + - `pex` (default): Use pip via PEX. + - `uv` (experimental): Pre-install dependencies into a uv venv, then pass it + to PEX via `--venv-repository`. When a PEX-native lockfile is available, + uv installs the exact pinned versions with `--no-deps`. + + Only applies to non-internal, non-cross-platform PEX builds. Other builds + silently fall back to pip. + """ + ), + advanced=True, + ) _resolves_to_interpreter_constraints = DictOption[list[str]]( help=softwrap( """ diff --git a/src/python/pants/backend/python/subsystems/uv.py b/src/python/pants/backend/python/subsystems/uv.py new file mode 100644 index 00000000000..c401f4d2190 --- /dev/null +++ b/src/python/pants/backend/python/subsystems/uv.py @@ -0,0 +1,82 @@ +# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from dataclasses import dataclass + +from pants.core.util_rules.external_tool import ( + TemplatedExternalTool, + download_external_tool, +) +from pants.engine.fs import Digest +from pants.engine.platform import Platform +from pants.engine.rules import collect_rules, rule +from pants.option.option_types import ArgsListOption +from pants.util.strutil import softwrap + + +class Uv(TemplatedExternalTool): + options_scope = "uv" + name = "uv" + help = "The uv Python package manager (https://github.com/astral-sh/uv)." + + default_version = "0.6.14" + default_known_versions = [ + "0.6.14|macos_x86_64|1d8ecb2eb3b68fb50e4249dc96ac9d2458dc24068848f04f4c5b42af2fd26552|16276555", + "0.6.14|macos_arm64|4ea4731010fbd1bc8e790e07f199f55a5c7c2c732e9b77f85e302b0bee61b756|15138933", + "0.6.14|linux_x86_64|0cac4df0cb3457b154f2039ae471e89cd4e15f3bd790bbb3cb0b8b40d940b93e|17032361", + "0.6.14|linux_arm64|94e22c4be44d205def456427639ca5ca1c1a9e29acc31808a7b28fdd5dcf7f17|15577079", + ] + version_constraints = ">=0.6.0,<1.0" + + default_url_template = ( + "https://github.com/astral-sh/uv/releases/download/{version}/uv-{platform}.tar.gz" + ) + default_url_platform_mapping = { + "linux_arm64": "aarch64-unknown-linux-musl", + "linux_x86_64": "x86_64-unknown-linux-musl", + "macos_arm64": "aarch64-apple-darwin", + "macos_x86_64": "x86_64-apple-darwin", + } + + def generate_exe(self, plat: Platform) -> str: + platform = self.default_url_platform_mapping[plat.value] + return f"./uv-{platform}/uv" + + args_for_uv_pip_install = ArgsListOption( + tool_name="uv", + example="--index-strategy unsafe-first-match", + extra_help=softwrap( + """ + Additional arguments to pass to `uv pip install` invocations. + + Used when `[python].pex_builder = "uv"` to pass extra flags to the + `uv pip install` step (e.g. `--index-url`, `--extra-index-url`). + These are NOT passed to the `uv venv` step. + """ + ), + ) + + +@dataclass(frozen=True) +class DownloadedUv: + """The downloaded uv binary with user-configured args.""" + + digest: Digest + exe: str + args_for_uv_pip_install: tuple[str, ...] + + +@rule +async def download_uv_binary(uv: Uv, platform: Platform) -> DownloadedUv: + downloaded = await download_external_tool(uv.get_request(platform)) + return DownloadedUv( + digest=downloaded.digest, + exe=downloaded.exe, + args_for_uv_pip_install=tuple(uv.args_for_uv_pip_install), + ) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index 67f63f22802..0dc5f9580d5 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -19,7 +19,9 @@ import packaging.version from packaging.requirements import Requirement -from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.subsystems import uv as uv_subsystem +from pants.backend.python.subsystems.setup import PexBuilder, PythonSetup +from pants.backend.python.subsystems.uv import download_uv_binary from pants.backend.python.target_types import ( Executable, MainSpecification, @@ -67,6 +69,7 @@ AddPrefix, CreateDigest, Digest, + Directory, FileContent, MergeDigests, RemovePrefix, @@ -82,6 +85,7 @@ add_prefix, create_digest, digest_to_snapshot, + get_digest_contents, merge_digests, remove_prefix, ) @@ -89,6 +93,7 @@ Process, ProcessCacheScope, ProcessResult, + execute_process_or_raise, fallible_to_exec_result_or_raise, ) from pants.engine.rules import collect_rules, concurrently, implicitly, rule @@ -492,6 +497,194 @@ async def _determine_pex_python_and_platforms(request: PexRequest) -> _BuildPexP ) +_UV_VENV_DIR = "__uv_venv" + + +@dataclass(frozen=True) +class _UvVenvRequest: + """Request to build a pre-populated venv using uv for PEX --venv-repository.""" + + req_strings: tuple[str, ...] + requirements: PexRequirements | EntireLockfile + python_path: str + description: str + + +@dataclass(frozen=True) +class _UvVenvResult: + """Result of building a uv venv.""" + + venv_digest: Digest | None + + +def _check_uv_preconditions( + request: PexRequest, + req_strings: tuple[str, ...], + pex_python_setup: _BuildPexPythonSetup, +) -> str | None: + """Check whether the uv builder can be used for this PEX request. + + Returns None if all preconditions are met, or a warning message describing + why uv cannot be used. + """ + label = request.description or request.output_filename + if not req_strings: + return ( + f"pex_builder=uv: no individual requirement strings for {label} " + "(e.g. using a whole-lockfile resolve or no third-party deps). " + "Falling back to the default PEX/pip builder." + ) + if request.platforms or request.complete_platforms: + return ( + f"pex_builder=uv: cross-platform build detected for {label}. " + "Falling back to the default PEX/pip builder." + ) + if pex_python_setup.python is None: + return ( + f"pex_builder=uv: no local Python interpreter available for {label}. " + "Falling back to the default PEX/pip builder." + ) + return None + + +@rule +async def _build_uv_venv( + uv_request: _UvVenvRequest, + pex_env: PexEnvironment, +) -> _UvVenvResult: + """Build a pre-populated venv using uv for use with PEX --venv-repository.""" + downloaded_uv = await download_uv_binary(**implicitly()) + + logger.debug( + "pex_builder=uv: using uv builder for %s", + uv_request.description, + ) + + # Try to extract the full resolved package list from the lockfile + # so we can pass pinned versions with --no-deps (reproducible). + # Fall back to letting uv resolve transitively if no lockfile. + all_resolved_reqs: tuple[str, ...] = () + if isinstance(uv_request.requirements, PexRequirements) and isinstance( + uv_request.requirements.from_superset, Resolve + ): + lockfile = await get_lockfile_for_resolve( + uv_request.requirements.from_superset, **implicitly() + ) + loaded_lockfile = await load_lockfile(LoadedLockfileRequest(lockfile), **implicitly()) + if loaded_lockfile.is_pex_native: + try: + digest_contents = await get_digest_contents(loaded_lockfile.lockfile_digest) + lockfile_bytes = next( + c.content for c in digest_contents if c.path == loaded_lockfile.lockfile_path + ) + lockfile_data = json.loads(lockfile_bytes) + all_resolved_reqs = tuple( + f"{req['project_name']}=={req['version']}" + for resolve in lockfile_data.get("locked_resolves", ()) + for req in resolve.get("locked_requirements", ()) + ) + except (json.JSONDecodeError, KeyError, StopIteration) as e: + logger.warning( + "pex_builder=uv: failed to parse lockfile for %s: %s. " + "Falling back to transitive uv resolution.", + uv_request.description, + e, + ) + all_resolved_reqs = () + + uv_reqs = all_resolved_reqs or uv_request.req_strings + + if all_resolved_reqs: + logger.debug( + "pex_builder=uv: using %d pinned packages from lockfile with --no-deps for %s", + len(all_resolved_reqs), + uv_request.description, + ) + else: + logger.debug( + "pex_builder=uv: no lockfile available, using transitive uv resolution for %s", + uv_request.description, + ) + + reqs_file = "__uv_requirements.txt" + reqs_content = "\n".join(uv_reqs) + "\n" + reqs_digest = await create_digest(CreateDigest([FileContent(reqs_file, reqs_content.encode())])) + + complete_pex_env = pex_env.in_sandbox(working_directory=None) + uv_cache_dir = ".cache/uv_cache" + uv_env = { + **complete_pex_env.environment_dict(python_configured=True), + "UV_CACHE_DIR": uv_cache_dir, + "UV_NO_CONFIG": "1", + } + uv_caches = { + **complete_pex_env.append_only_caches, + "uv_cache": uv_cache_dir, + } + uv_tmpdir = "__uv_tmp" + tmpdir_digest = await create_digest(CreateDigest([Directory(uv_tmpdir)])) + + python_path = uv_request.python_path + + uv_input = await merge_digests(MergeDigests([downloaded_uv.digest, reqs_digest, tmpdir_digest])) + + # Step 1: Create venv with uv. + venv_result = await execute_process_or_raise( + **implicitly( + Process( + argv=( + downloaded_uv.exe, + "venv", + _UV_VENV_DIR, + "--python", + python_path, + ), + input_digest=uv_input, + output_directories=(_UV_VENV_DIR,), + env={**uv_env, "TMPDIR": uv_tmpdir}, + append_only_caches=uv_caches, + description=f"Create uv venv for {uv_request.description}", + level=LogLevel.DEBUG, + cache_scope=ProcessCacheScope.SUCCESSFUL, + ) + ) + ) + + # Step 2: Install dependencies into the venv. + install_input = await merge_digests(MergeDigests([uv_input, venv_result.output_digest])) + + install_argv: tuple[str, ...] = ( + downloaded_uv.exe, + "pip", + "install", + "--python", + os.path.join(_UV_VENV_DIR, "bin", "python"), + "-r", + reqs_file, + *(("--no-deps",) if all_resolved_reqs else ()), + *downloaded_uv.args_for_uv_pip_install, + ) + + uv_install_result = await execute_process_or_raise( + **implicitly( + Process( + argv=install_argv, + input_digest=install_input, + output_directories=(_UV_VENV_DIR,), + env={**uv_env, "TMPDIR": uv_tmpdir}, + append_only_caches=uv_caches, + description=f"uv pip install for {uv_request.description}", + level=LogLevel.DEBUG, + cache_scope=ProcessCacheScope.SUCCESSFUL, + ) + ) + ) + + return _UvVenvResult( + venv_digest=uv_install_result.output_digest, + ) + + @dataclass class _BuildPexRequirementsSetup: digests: list[Digest] @@ -702,7 +895,10 @@ async def _setup_pex_requirements( @rule(level=LogLevel.DEBUG) async def build_pex( - request: PexRequest, python_setup: PythonSetup, pex_subsystem: PexSubsystem + request: PexRequest, + python_setup: PythonSetup, + pex_subsystem: PexSubsystem, + pex_env: PexEnvironment, ) -> BuildPexResult: """Returns a PEX with the given settings.""" @@ -756,6 +952,45 @@ async def build_pex( ) req_strings = () + # Experimental: build PEX via uv + --venv-repository. + # When opted in, we use uv to create a pre-populated venv and let PEX + # package from it instead of resolving with pip. + # See: https://github.com/pantsbuild/pants/issues/20679 + uv_venv_digest: Digest | None = None + + use_uv_builder = python_setup.pex_builder == PexBuilder.uv + # uv builder only applies to non-internal PEXes with requirements and a + # local interpreter (not cross-platform builds). + if use_uv_builder and not request.internal_only: + fallback_reason = _check_uv_preconditions(request, req_strings, pex_python_setup) + if fallback_reason: + logger.warning(fallback_reason) + else: + assert pex_python_setup.python is not None + uv_result = await _build_uv_venv( + _UvVenvRequest( + req_strings=req_strings, + requirements=request.requirements, + python_path=pex_python_setup.python.path, + description=request.description or request.output_filename, + ), + **implicitly(), + ) + uv_venv_digest = uv_result.venv_digest + + # Replace requirements_setup: pass requirement strings + --venv-repository + # so PEX subsets from the uv-populated venv instead of resolving with pip. + requirements_setup = _BuildPexRequirementsSetup( + digests=[], + argv=[*req_strings, f"--venv-repository={_UV_VENV_DIR}"], + concurrency_available=requirements_setup.concurrency_available, + ) + elif use_uv_builder and request.internal_only: + logger.debug( + "pex_builder=uv: skipping for internal-only PEX %s. Using the default PEX/pip builder.", + request.description or request.output_filename, + ) + output_chroot = os.path.dirname(request.output_filename) if output_chroot: output_file = request.output_filename @@ -776,7 +1011,13 @@ async def build_pex( *request.additional_args, ] - argv.extend(pex_python_setup.argv) + if uv_venv_digest is not None: + # When using --venv-repository, PEX does not allow any custom target + # flags (--python, --interpreter-constraint, --platform). The target is + # implicitly the venv interpreter. + pass + else: + argv.extend(pex_python_setup.argv) if request.main is not None: argv.extend(request.main.iter_pex_args()) @@ -819,6 +1060,7 @@ async def build_pex( request.additional_inputs, *requirements_setup.digests, *(pex.digest for pex in request.pex_path), + *([uv_venv_digest] if uv_venv_digest else []), ) ) ) @@ -1457,4 +1699,10 @@ async def determine_pex_resolve_info(pex_pex: PexPEX, pex: Pex) -> PexResolveInf def rules(): - return [*collect_rules(), *pex_cli.rules(), *pex_requirements.rules(), *stripped_source_rules()] + return [ + *collect_rules(), + *pex_cli.rules(), + *pex_requirements.rules(), + *uv_subsystem.rules(), # Also in register.py; engine deduplicates. + *stripped_source_rules(), + ] diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index 410a5b5fc9a..2a23116b14f 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -969,3 +969,118 @@ def test_digest_complete_platforms_codegen(rule_runner: RuleRunner) -> None: # Verify the result assert len(complete_platforms) == 1 assert complete_platforms.digest != EMPTY_DIGEST + + +def test_uv_pex_builder_resolves_dependencies(rule_runner: RuleRunner) -> None: + """When pex_builder=uv, PEX should be built via uv venv + --venv-repository.""" + req_strings = ["six==1.12.0", "jsonschema==2.6.0"] + requirements = PexRequirements(req_strings) + pex_info = create_pex_and_get_pex_info( + rule_runner, + requirements=requirements, + additional_pants_args=("--python-pex-builder=uv",), + internal_only=False, + ) + assert set(parse_requirements(req_strings)).issubset( + set(parse_requirements(pex_info["requirements"])) + ) + + +def test_uv_pex_builder_includes_transitive_dependencies(rule_runner: RuleRunner) -> None: + """uv builder must install transitive dependencies, not just direct ones. + + `requests` depends on urllib3, certifi, charset-normalizer, idna - the PEX + must be able to import these at runtime even though only `requests` is declared. + We verify by actually executing the PEX and importing a transitive dep. + """ + sources = rule_runner.request( + Digest, + [ + CreateDigest( + ( + FileContent( + "main.py", + # Import both the direct dep and a transitive dep (certifi). + b"import requests; import certifi; print(f'requests=={requests.__version__}'); print(f'certifi_where={certifi.where()}')", + ), + ) + ), + ], + ) + pex_data = create_pex_and_get_all_data( + rule_runner, + pex_type=Pex, + requirements=PexRequirements(["requests==2.31.0"]), + main=EntryPoint("main"), + sources=sources, + additional_pants_args=("--python-pex-builder=uv",), + internal_only=False, + ) + pex_exe = ( + f"./{pex_data.sandbox_path}" + if pex_data.is_zipapp + else os.path.join(pex_data.sandbox_path, "__main__.py") + ) + process = Process( + argv=(pex_exe,), + env={"PATH": os.getenv("PATH", "")}, + input_digest=pex_data.pex.digest, + description="Run uv-built pex and verify transitive deps are importable", + ) + result = rule_runner.request(ProcessResult, [process]) + assert b"requests==2.31.0" in result.stdout + assert b"certifi_where=" in result.stdout + + +def test_uv_pex_builder_execution(rule_runner: RuleRunner) -> None: + """PEX built via uv builder should actually execute and import installed packages.""" + sources = rule_runner.request( + Digest, + [ + CreateDigest( + ( + FileContent( + "main.py", + b"import six; print(f'six=={six.__version__}')", + ), + ) + ), + ], + ) + pex_data = create_pex_and_get_all_data( + rule_runner, + pex_type=Pex, + requirements=PexRequirements(["six==1.12.0"]), + main=EntryPoint("main"), + sources=sources, + additional_pants_args=("--python-pex-builder=uv",), + internal_only=False, + ) + pex_exe = ( + f"./{pex_data.sandbox_path}" + if pex_data.is_zipapp + else os.path.join(pex_data.sandbox_path, "__main__.py") + ) + process = Process( + argv=(pex_exe,), + env={"PATH": os.getenv("PATH", "")}, + input_digest=pex_data.pex.digest, + description="Run uv-built pex and verify import works", + ) + result = rule_runner.request(ProcessResult, [process]) + assert result.stdout == b"six==1.12.0\n" + + +def test_uv_pex_builder_skipped_for_internal_only(rule_runner: RuleRunner) -> None: + """Internal-only PEXes should fall back to the default pip path even with pex_builder=uv.""" + req_strings = ["six==1.12.0"] + requirements = PexRequirements(req_strings) + pex_info = create_pex_and_get_pex_info( + rule_runner, + requirements=requirements, + additional_pants_args=("--python-pex-builder=uv",), + internal_only=True, + ) + assert set(parse_requirements(req_strings)).issubset( + set(parse_requirements(pex_info["requirements"])) + )