diff --git a/src/fromager/gitutils.py b/src/fromager/gitutils.py index daa63b68..7b5f2aaa 100644 --- a/src/fromager/gitutils.py +++ b/src/fromager/gitutils.py @@ -1,10 +1,16 @@ +from __future__ import annotations + import logging import pathlib +import typing from urllib.parse import urlparse from packaging.requirements import Requirement -from fromager import context, external_commands +from . import external_commands + +if typing.TYPE_CHECKING: + from . import context logger = logging.getLogger(__name__) diff --git a/src/fromager/metrics.py b/src/fromager/metrics.py index b7bab031..144025f6 100644 --- a/src/fromager/metrics.py +++ b/src/fromager/metrics.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import functools import logging import time @@ -7,7 +9,8 @@ from packaging.requirements import Requirement from packaging.version import Version -from . import context +if typing.TYPE_CHECKING: + from . import context def timeit(description: str) -> typing.Callable: diff --git a/src/fromager/packagesettings.py b/src/fromager/packagesettings.py index 46355419..49efb124 100644 --- a/src/fromager/packagesettings.py +++ b/src/fromager/packagesettings.py @@ -16,7 +16,7 @@ from pydantic import Field from pydantic_core import CoreSchema, core_schema -from . import overrides +from . import overrides, sources if typing.TYPE_CHECKING: from . import build_environment @@ -197,6 +197,15 @@ class BuildOptions(pydantic.BaseModel): 0.5: assume each parallel job requires 512 MB virtual memory """ + sdist_backend: sources.SDistBackend = Field(default=sources.SDistBackend.PEP517) + """sdist build backend + + - use PEP 517 build backend by default + - 'tarball' just tars the source directory + - projects with non-standard "build_dir" use 'tarball' automatically + - projects with 'build_sdist' hook ignore this setting + """ + class ProjectOverride(pydantic.BaseModel): """Override pyproject.toml settings @@ -732,6 +741,14 @@ def build_ext_parallel(self) -> bool: """Configure [build_ext]parallel for setuptools?""" return self._ps.build_options.build_ext_parallel + @property + def sdist_backend(self) -> sources.SDistBackend: + """Source distribution backend""" + if self._ps.build_dir is not None: + # always use tarball for projects with non-standard build_dir + return sources.SDistBackend.TARBALL + return self._ps.build_options.sdist_backend + @property def config_settings(self) -> list[str]: return self._ps.config_settings diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 5e8f322f..840fbf9a 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -1,5 +1,7 @@ from __future__ import annotations +import enum +import errno import inspect import json import logging @@ -38,6 +40,13 @@ logger = logging.getLogger(__name__) +class SDistBackend(enum.StrEnum): + # build sdist with PEP 517 hook + PEP517 = "pep571" + # create sdist with tarfile + TARBALL = "tarball" + + def get_source_type(ctx: context.WorkContext, req: Requirement) -> str: source_type = requirements_file.SourceType.SDIST pbi = ctx.package_build_info(req) @@ -631,6 +640,11 @@ def build_sdist( return sdist_filename +# meson dist currently only works with Git or Mercurial repos +# affects NumPy and projects with https://mesonbuild.com/meson-python/ +BUILD_BACKEND_FORCE_TARBALL = {"mesonpy"} + + def default_build_sdist( ctx: context.WorkContext, extra_environ: dict, @@ -639,6 +653,50 @@ def default_build_sdist( sdist_root_dir: pathlib.Path, build_env: build_environment.BuildEnvironment, build_dir: pathlib.Path, +) -> pathlib.Path: + pbi = ctx.package_build_info(req) + sdist_backend = pbi.sdist_backend + + pyproject_toml = dependencies.get_pyproject_contents(sdist_root_dir) + backend_settings = dependencies.get_build_backend(pyproject_toml) + build_backend = backend_settings["build-backend"] + if build_backend in BUILD_BACKEND_FORCE_TARBALL: + logger.info( + f"{req.name}: force build sdist tarball for backend '{build_backend}'" + ) + sdist_backend = SDistBackend.TARBALL + + match sdist_backend: + case SDistBackend.PEP517: + return pep517_build_sdist( + ctx=ctx, + extra_environ=extra_environ, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + ) + case SDistBackend.TARBALL: + return tarball_build_sdist( + ctx=ctx, + extra_environ=extra_environ, + req=req, + version=version, + sdist_root_dir=sdist_root_dir, + build_env=build_env, + build_dir=build_dir, + ) + case _ as unreachable: + typing.assert_never(unreachable) + + +def tarball_build_sdist( + ctx: context.WorkContext, + extra_environ: dict, + req: Requirement, + version: Version, + sdist_root_dir: pathlib.Path, + build_env: build_environment.BuildEnvironment, + build_dir: pathlib.Path, ) -> pathlib.Path: # It seems like the "correct" way to do this would be to run the # PEP 517 API in the source tree we have modified. However, quite @@ -650,6 +708,7 @@ def default_build_sdist( # # For cases where the PEP 517 approach works, use # pep517_build_sdist(). + logger.debug(f"{req.name}: build sdist tarball") sdist_filename = ctx.sdists_builds / f"{req.name}-{version}.tar.gz" if sdist_filename.exists(): sdist_filename.unlink() @@ -670,14 +729,24 @@ def pep517_build_sdist( req: Requirement, sdist_root_dir: pathlib.Path, version: Version, + build_env: build_environment.BuildEnvironment | None = None, ) -> pathlib.Path: """Use the PEP 517 API to build a source distribution from a modified source tree.""" + logger.debug(f"{req.name}: build sdist PEP 517") pyproject_toml = dependencies.get_pyproject_contents(sdist_root_dir) hook_caller = dependencies.get_build_backend_hook_caller( sdist_root_dir, pyproject_toml, extra_environ, network_isolation=ctx.network_isolation, + build_env=build_env, ) sdist_filename = hook_caller.build_sdist(ctx.sdists_builds) - return ctx.sdists_builds / sdist_filename + sdist = ctx.sdists_builds / sdist_filename + if not sdist.is_file(): + raise FileNotFoundError( + errno.ENOENT, + f"{req.name}: PEP 517 build sdist failed for {req.name}=={version}", + sdist, + ) + return sdist diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 3be88ede..52ef48d2 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -7,7 +7,7 @@ from packaging.utils import NormalizedName from packaging.version import Version -from fromager import build_environment, context +from fromager import build_environment, context, sources from fromager.packagesettings import ( BuildDirectory, EnvVars, @@ -29,6 +29,7 @@ "build_ext_parallel": True, "cpu_cores_per_job": 4, "memory_per_job_gb": 4.0, + "sdist_backend": sources.SDistBackend.TARBALL, }, "changelog": { Version("1.0.1"): ["fixed bug"], @@ -85,6 +86,7 @@ "build_ext_parallel": False, "cpu_cores_per_job": 1, "memory_per_job_gb": 1.0, + "sdist_backend": sources.SDistBackend.PEP517, }, "changelog": {}, "config_settings": [], diff --git a/tests/testdata/context/overrides/settings/test_pkg.yaml b/tests/testdata/context/overrides/settings/test_pkg.yaml index 3276dc62..083718fa 100644 --- a/tests/testdata/context/overrides/settings/test_pkg.yaml +++ b/tests/testdata/context/overrides/settings/test_pkg.yaml @@ -3,6 +3,7 @@ build_options: build_ext_parallel: true cpu_cores_per_job: 4 memory_per_job_gb: 4 + sdist_backend: tarball changelog: "1.0.1": - fixed bug