Add sdist_backend setting, default to PEP 517

src/fromager/gitutils.py

@@ -1,10 +1,16 @@
+from __future__ import annotations
+
 import logging
 import pathlib
+import typing
 from urllib.parse import urlparse
 
 from packaging.requirements import Requirement
 
-from fromager import context, external_commands
+from . import external_commands
+
+if typing.TYPE_CHECKING:
+    from . import context
 
 logger = logging.getLogger(__name__)
 

src/fromager/metrics.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import functools
 import logging
 import time
@@ -7,7 +9,8 @@
 from packaging.requirements import Requirement
 from packaging.version import Version
 
-from . import context
+if typing.TYPE_CHECKING:
+    from . import context
 
 
 def timeit(description: str) -> typing.Callable:

src/fromager/packagesettings.py

@@ -16,7 +16,7 @@
 from pydantic import Field
 from pydantic_core import CoreSchema, core_schema
 
-from . import overrides
+from . import overrides, sources
 
 if typing.TYPE_CHECKING:
     from . import build_environment
@@ -197,6 +197,15 @@ class BuildOptions(pydantic.BaseModel):
     0.5: assume each parallel job requires 512 MB virtual memory
     """
 
+    sdist_backend: sources.SDistBackend = Field(default=sources.SDistBackend.PEP517)
+    """sdist build backend
+
+    - use PEP 517 build backend by default
+    - 'tarball' just tars the source directory
+    - projects with non-standard "build_dir" use 'tarball' automatically
+    - projects with 'build_sdist' hook ignore this setting
+    """
+
 
 class ProjectOverride(pydantic.BaseModel):
     """Override pyproject.toml settings
@@ -732,6 +741,14 @@ def build_ext_parallel(self) -> bool:
         """Configure [build_ext]parallel for setuptools?"""
         return self._ps.build_options.build_ext_parallel
 
+    @property
+    def sdist_backend(self) -> sources.SDistBackend:
+        """Source distribution backend"""
+        if self._ps.build_dir is not None:
+            # always use tarball for projects with non-standard build_dir
+            return sources.SDistBackend.TARBALL
+        return self._ps.build_options.sdist_backend
+
     @property
     def config_settings(self) -> list[str]:
         return self._ps.config_settings

src/fromager/sources.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import enum
+import errno
 import inspect
 import json
 import logging
@@ -38,6 +40,13 @@
 logger = logging.getLogger(__name__)
 
 
+class SDistBackend(enum.StrEnum):
+    # build sdist with PEP 517 hook
+    PEP517 = "pep571"
+    # create sdist with tarfile
+    TARBALL = "tarball"
+
+
 def get_source_type(ctx: context.WorkContext, req: Requirement) -> str:
     source_type = requirements_file.SourceType.SDIST
     pbi = ctx.package_build_info(req)
@@ -631,6 +640,11 @@ def build_sdist(
     return sdist_filename
 
 
+# meson dist currently only works with Git or Mercurial repos
+# affects NumPy and projects with https://mesonbuild.com/meson-python/
+BUILD_BACKEND_FORCE_TARBALL = {"mesonpy"}
+
+
 def default_build_sdist(
     ctx: context.WorkContext,
     extra_environ: dict,
@@ -639,6 +653,50 @@ def default_build_sdist(
     sdist_root_dir: pathlib.Path,
     build_env: build_environment.BuildEnvironment,
     build_dir: pathlib.Path,
+) -> pathlib.Path:
+    pbi = ctx.package_build_info(req)
+    sdist_backend = pbi.sdist_backend
+
+    pyproject_toml = dependencies.get_pyproject_contents(sdist_root_dir)
+    backend_settings = dependencies.get_build_backend(pyproject_toml)
+    build_backend = backend_settings["build-backend"]
+    if build_backend in BUILD_BACKEND_FORCE_TARBALL:
+        logger.info(
+            f"{req.name}: force build sdist tarball for backend '{build_backend}'"
+        )
+        sdist_backend = SDistBackend.TARBALL
+
+    match sdist_backend:
+        case SDistBackend.PEP517:
+            return pep517_build_sdist(
+                ctx=ctx,
+                extra_environ=extra_environ,
+                req=req,
+                version=version,
+                sdist_root_dir=sdist_root_dir,
+            )
+        case SDistBackend.TARBALL:
+            return tarball_build_sdist(
+                ctx=ctx,
+                extra_environ=extra_environ,
+                req=req,
+                version=version,
+                sdist_root_dir=sdist_root_dir,
+                build_env=build_env,
+                build_dir=build_dir,
+            )
+        case _ as unreachable:
+            typing.assert_never(unreachable)
+
+
+def tarball_build_sdist(
+    ctx: context.WorkContext,
+    extra_environ: dict,
+    req: Requirement,
+    version: Version,
+    sdist_root_dir: pathlib.Path,
+    build_env: build_environment.BuildEnvironment,
+    build_dir: pathlib.Path,
 ) -> pathlib.Path:
     # It seems like the "correct" way to do this would be to run the
     # PEP 517 API in the source tree we have modified. However, quite
@@ -650,6 +708,7 @@ def default_build_sdist(
     #
     # For cases where the PEP 517 approach works, use
     # pep517_build_sdist().
+    logger.debug(f"{req.name}: build sdist tarball")
     sdist_filename = ctx.sdists_builds / f"{req.name}-{version}.tar.gz"
     if sdist_filename.exists():
         sdist_filename.unlink()
@@ -670,14 +729,24 @@ def pep517_build_sdist(
     req: Requirement,
     sdist_root_dir: pathlib.Path,
     version: Version,
+    build_env: build_environment.BuildEnvironment | None = None,
 ) -> pathlib.Path:
     """Use the PEP 517 API to build a source distribution from a modified source tree."""
+    logger.debug(f"{req.name}: build sdist PEP 517")
     pyproject_toml = dependencies.get_pyproject_contents(sdist_root_dir)
     hook_caller = dependencies.get_build_backend_hook_caller(
         sdist_root_dir,
         pyproject_toml,
         extra_environ,
         network_isolation=ctx.network_isolation,
+        build_env=build_env,
     )
     sdist_filename = hook_caller.build_sdist(ctx.sdists_builds)
-    return ctx.sdists_builds / sdist_filename
+    sdist = ctx.sdists_builds / sdist_filename
+    if not sdist.is_file():
+        raise FileNotFoundError(
+            errno.ENOENT,
+            f"{req.name}: PEP 517 build sdist failed for {req.name}=={version}",
+            sdist,
+        )
+    return sdist

tests/test_packagesettings.py

@@ -7,7 +7,7 @@
 from packaging.utils import NormalizedName
 from packaging.version import Version
 
-from fromager import build_environment, context
+from fromager import build_environment, context, sources
 from fromager.packagesettings import (
     BuildDirectory,
     EnvVars,
@@ -29,6 +29,7 @@
         "build_ext_parallel": True,
         "cpu_cores_per_job": 4,
         "memory_per_job_gb": 4.0,
+        "sdist_backend": sources.SDistBackend.TARBALL,
     },
     "changelog": {
         Version("1.0.1"): ["fixed bug"],
@@ -85,6 +86,7 @@
         "build_ext_parallel": False,
         "cpu_cores_per_job": 1,
         "memory_per_job_gb": 1.0,
+        "sdist_backend": sources.SDistBackend.PEP517,
     },
     "changelog": {},
     "config_settings": [],

tests/testdata/context/overrides/settings/test_pkg.yaml

@@ -3,6 +3,7 @@ build_options:
     build_ext_parallel: true
     cpu_cores_per_job: 4
     memory_per_job_gb: 4
+    sdist_backend: tarball
 changelog:
     "1.0.1":
         - fixed bug