Merge pull request #3 from TheJJ/build-improvements

mikonse · web-flow · commit b8e357dfb77b · 2025-11-24T23:12:52.000+01:00
build improvements
diff --git a/pyproject.toml b/pyproject.toml
@@ -31,11 +31,14 @@ exclude_gitignore = true
 [tool.ruff]
 line-length = 120
 target-version = "py312"
-exclude = [".git", ".idea", ".mypy_cache", ".venv*", "docs", "debian"]
+extend-exclude = [".idea", ".mypy_cache", ".venv*", "docs", "debian", "__pycache__", "*.egg_info"]
 
 [tool.ruff.lint]
-select = ["E4", "E7", "E9", "F", "I", "PLE", "PLW"]
-ignore = ["E722"]
+select = ["E", "W", "F", "I", "C", "N", "PL", "RUF", "I001"]
+ignore = ["E722", "PLR2004", "PLR0912", "PLR5501", "PLC0415"]
+mccabe.max-complexity = 25
+pylint.max-args = 10
 
 [tool.pytest]
 testpaths = ["tests"]
+addopts = ["--ignore=tests/integration/packages"]
diff --git a/src/debmagic/_build.py b/src/debmagic/_build.py
@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-import shlex
 import typing
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import Sequence
 
 from ._build_stage import BuildStage
 from ._utils import run_cmd
@@ -29,21 +29,13 @@ class Build:
     dry_run: bool = False
 
     _completed_stages: set[BuildStage] = field(default_factory=set)
-    _selected_packages: list[BinaryPackage] | None = field(default_factory=list)
 
-    def cmd(self, cmd: list[str] | str, **kwargs):
+    def cmd(self, cmd: Sequence[str] | str, **kwargs):
         """
         execute a command, auto-converts command strings/lists.
         use this to supports build dry-runs.
         """
-        cmd_args: list[str] | str = cmd
-        is_shell = kwargs.get("shell")
-        if not is_shell and isinstance(cmd, str):
-            cmd_args = shlex.split(cmd)
-        elif is_shell and not isinstance(cmd, str):
-            cmd_args = shlex.join(cmd)
-
-        run_cmd(cmd_args, dry_run=self.dry_run, **kwargs)
+        run_cmd(cmd, dry_run=self.dry_run, **kwargs)
 
     @property
     def install_dirs(self) -> dict[str, Path]:
@@ -52,17 +44,20 @@ def install_dirs(self) -> dict[str, Path]:
 
     def select_packages(self, names: set[str]):
         """only build those packages"""
-        if not names:
-            self._selected_packages = None
+        self.binary_packages = []
 
-        self._selected_packages = list()
-        for pkg in self.binary_packages:
+        for pkg in self.source_package.binary_packages:
             if pkg.name in names:
-                self._selected_packages.append(pkg)
+                self.binary_packages.append(pkg)
 
     def filter_packages(self, package_filter: PackageFilter) -> None:
         """apply filter to only build those packages"""
-        self.select_packages({pkg.name for pkg in package_filter.get_packages(self.binary_packages)})
+        self.select_packages({pkg.name for pkg in package_filter.get_packages(self.source_package.binary_packages)})
+
+    def filtered_binary_packages(self, names: set[str]) -> typing.Iterator[BinaryPackage]:
+        for pkg in self.binary_packages:
+            if pkg.name in names:
+                yield pkg
 
     def is_stage_completed(self, stage: BuildStage) -> bool:
         return stage in self._completed_stages
@@ -95,13 +90,12 @@ def run(
                 for preset in self.presets:
                     print(f"debmagic:  trying preset {preset}...")
                     if preset_stage_function := preset.get_stage(stage):
-                        print("debmagic:  preset has function")
+                        print("debmagic:   running stage from preset")
                         preset_stage_function(self)
                         self._mark_stage_done(stage)
                         break  # stop preset processing
 
             if not self.is_stage_completed(stage):
-                breakpoint()
                 raise RuntimeError(f"{stage!s} stage was never executed")
 
             if stage == target_stage:
diff --git a/src/debmagic/_module/autodetect.py b/src/debmagic/_module/autodetect.py
diff --git a/src/debmagic/_module/autotools.py b/src/debmagic/_module/autotools.py
@@ -47,16 +47,16 @@ def configure(self, build: Build, args: list[str] | None = None):
         # TODO: show some config.log if configure failed
 
     def build(self, build: Build, args: list[str] = []):
-        args = [f"-j{build.parallel}"] + args
+        args = [f"-j{build.parallel}", *args]
 
-        build.cmd(["make"] + args, cwd=build.source_dir)
+        build.cmd(["make", *args], cwd=build.source_dir)
 
     # TODO: def test(): run make test or make check
 
     def install(self, build: Build, args: list[str] = []):
         # TODO: figure out installdir handling for multi package builds
         destdir = build.install_dirs[build.source_package.name]
-        build.cmd(["make", f"DESTDIR={destdir}", "install"] + args, cwd=build.source_dir)
+        build.cmd(["make", f"DESTDIR={destdir}", "install", *args], cwd=build.source_dir)
 
 
 def autoreconf(build: Build):
diff --git a/src/debmagic/_module/default.py b/src/debmagic/_module/default.py
diff --git a/src/debmagic/_module/dh.py b/src/debmagic/_module/dh.py
@@ -12,14 +12,17 @@
 from .._build import Build
 from .._package import SourcePackage
 from .._preset import Preset as PresetBase
-from .._utils import prefix_idx, run_cmd
+from .._utils import list_strip_head, prefix_idx, run_cmd
 
 
 class DHSequenceID(StrEnum):
     clean = "clean"
     build = "build"
     build_arch = "build-arch"
     build_indep = "build-indep"
+    install = "install"
+    install_arch = "install-arch"
+    install_indep = "install-indep"
     binary = "binary"
     binary_arch = "binary-arch"
     binary_indep = "binary-indep"
@@ -33,16 +36,16 @@ def __init__(self, dh_invocation: str | None = None):
         if not dh_invocation:
             dh_invocation = "dh"
         self._dh_invocation: str = dh_invocation
-        self._overrides: dict[str, DHOverride] = dict()
+        self._overrides: dict[str, DHOverride] = {}
         self._initialized = False
 
         # debmagic's stages, with matching commands from the dh sequence
-        self._clean_seq: list[str] = list()
-        self._configure_seq: list[str] = list()
-        self._build_seq: list[str] = list()
-        self._test_seq: list[str] = list()
-        self._install_seq: list[str] = list()
-        self._package_seq: list[str] = list()
+        self._clean_seq: list[str] = []
+        self._configure_seq: list[str] = []
+        self._build_seq: list[str] = []
+        self._test_seq: list[str] = []
+        self._install_seq: list[str] = []
+        self._package_seq: list[str] = []
 
         # all seen sequence cmd ids (the dh command script itself)
         self._seq_ids: set[str] = set()
@@ -106,10 +109,10 @@ def _populate_stages(self, dh_invocation: str, base_dir: Path) -> None:
         self._clean_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.clean)
 
         ## untangle "build" to configure & build & test
-        build_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.build)
-        if build_seq[-1] != "create-stamp debian/debhelper-build-stamp":
+        build_seq_raw = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.build)
+        if build_seq_raw[-1] != "create-stamp debian/debhelper-build-stamp":
             raise RuntimeError("build stamp creation line missing from dh build sequence")
-        build_seq = build_seq[:-1]  # remove that stamp line
+        build_seq = build_seq_raw[:-1]  # remove that stamp line
 
         auto_cfg_idx = prefix_idx("dh_auto_configure", build_seq)
         # up to including dh_auto_configure
@@ -125,14 +128,12 @@ def _populate_stages(self, dh_invocation: str, base_dir: Path) -> None:
         self._test_seq = build_seq[auto_test_idx:]
 
         ## untangle "binary" to install & package
+        install_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.install)
+        self._install_seq = list_strip_head(install_seq, build_seq_raw)
         binary_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.binary)
-        build_stamp_idx = prefix_idx("create-stamp debian/debhelper-build-stamp", binary_seq)
-        auto_install_idx = prefix_idx("dh_auto_install", binary_seq)
-        # one after the build-stamp, up to including dh_auto_install
-        self._install_seq = binary_seq[build_stamp_idx + 1 : auto_install_idx + 1]
-        # assume everything else is packing (which is a bit wrong, but packing & installing is mixed in dh)
-        self._package_seq = binary_seq[auto_install_idx + 1 :]
+        self._package_seq = list_strip_head(binary_seq, install_seq)
 
+        # register all sequence items for validity checks
         for seq in (
             self._clean_seq,
             self._configure_seq,
@@ -148,7 +149,7 @@ def _populate_stages(self, dh_invocation: str, base_dir: Path) -> None:
 
     def _get_dh_seq(self, base_dir: Path, dh_invocation: str, seq: DHSequenceID) -> list[str]:
         dh_base_cmd = shlex.split(dh_invocation)
-        cmd = dh_base_cmd + [str(seq), "--no-act"]
+        cmd = [*dh_base_cmd, str(seq), "--no-act"]
         proc = run_cmd(cmd, cwd=base_dir, capture_output=True, text=True)
         lines = proc.stdout.splitlines()
         return [line.strip() for line in lines]
diff --git a/src/debmagic/_module/python.py b/src/debmagic/_module/python.py
diff --git a/src/debmagic/_package.py b/src/debmagic/_package.py
@@ -95,7 +95,7 @@ class PackageFilter(Flag):
     architecture_independent = auto()
 
     def get_packages(self, binary_packages: list[BinaryPackage]) -> list[BinaryPackage]:
-        ret: list[BinaryPackage] = list()
+        ret: list[BinaryPackage] = []
 
         for pkg in binary_packages:
             if self.architecture_independent and pkg.arch_dependent:
@@ -157,7 +157,7 @@ def something(arg: str = 'stuff'):
 
         # find arguments and its types to guess argparsing
         args_raw = inspect.getfullargspec(func)
-        args: CustomFuncArgsT = dict()
+        args: CustomFuncArgsT = {}
         default_count = 0 if not args_raw.defaults else len(args_raw.defaults)
         default_start = len(args_raw.args) - default_count
         for idx, arg in enumerate(args_raw.args):
@@ -253,7 +253,7 @@ def package(
     presets: list[Preset] = as_presets(preset)
 
     # apply default preset last
-    from ._modules.default import Preset as DefaultPreset
+    from ._module.default import Preset as DefaultPreset
 
     presets.append(DefaultPreset())
 
@@ -263,7 +263,7 @@ def package(
 
     src_pkg: SourcePackage | None = None
     # which binary packages should be produced?
-    bin_pkgs: list[BinaryPackage] = list()
+    bin_pkgs: list[BinaryPackage] = []
 
     for block in deb822.DebControl.iter_paragraphs(
         (rules_file.package_dir / "debian/control").open(),
diff --git a/src/debmagic/_preset.py b/src/debmagic/_preset.py
@@ -96,7 +96,7 @@ def _get_member(obj: T | type[T], stage: BuildStage) -> BuildStep | _PresetBuild
 
 
 def as_presets(preset_elements: PresetsT) -> list[Preset]:
-    presets: list[Preset] = list()
+    presets: list[Preset] = []
 
     if isinstance(preset_elements, list):
         for preset_module in preset_elements:
diff --git a/src/debmagic/_rules_file.py b/src/debmagic/_rules_file.py
@@ -13,7 +13,7 @@ def find_rules_file() -> RulesFile:
     for frame in inspect.stack():
         file_path = Path(frame.filename)
         # TODO further validation, be in debian/
-        if file_path.name == "rules" or file_path.name == "rules.py":
+        if file_path.name in {"rules", "rules.py"}:
             return RulesFile(
                 package_dir=file_path.parent.parent.resolve(),
                 local_vars=frame.frame.f_locals,
diff --git a/src/debmagic/_utils.py b/src/debmagic/_utils.py
@@ -3,6 +3,7 @@
 import shlex
 import subprocess
 import sys
+from typing import Sequence, TypeVar
 
 
 class Namespace:
@@ -11,8 +12,8 @@ class Namespace:
     """
 
     def __init__(self, **kwargs):
-        for name in kwargs:
-            setattr(self, name, kwargs[name])
+        for name, value in kwargs.items():
+            setattr(self, name, value)
 
     def __eq__(self, other):
         if not isinstance(other, Namespace):
@@ -35,20 +36,33 @@ def get(self, key):
         return self.__dict__.get(key)
 
 
-def run_cmd(args: list[str] | str, check: bool = True, dry_run: bool = False, **kwargs) -> subprocess.CompletedProcess:
-    shell = kwargs.get("shell")
-    if shell:
-        print(f"debmagic: {args}")
+def run_cmd(
+    cmd: Sequence[str] | str,
+    check: bool = True,
+    dry_run: bool = False,
+    **kwargs,
+) -> subprocess.CompletedProcess:
+    cmd_args: Sequence[str] | str = cmd
+    cmd_pretty: str
+
+    if kwargs.get("shell"):
+        if not isinstance(cmd, str):
+            cmd_args = shlex.join(cmd)
+    else:
+        if isinstance(cmd, str):
+            cmd_args = shlex.split(cmd)
+
+    if isinstance(cmd, str):
+        cmd_pretty = cmd
     else:
-        if not isinstance(args, (list, tuple)):
-            raise ValueError("need list/tuple as command arguments when not using shell=True")
-        cmd_pretty = shlex.join(args)
-        print(f"debmagic: {cmd_pretty}")
+        cmd_pretty = shlex.join(cmd_args)
+
+    print(f"debmagic: {cmd_pretty}")
 
     if dry_run:
-        return subprocess.CompletedProcess(args, 0)
+        return subprocess.CompletedProcess(cmd_args, 0)
 
-    ret = subprocess.run(args, check=False, **kwargs)
+    ret = subprocess.run(cmd_args, check=False, **kwargs)
 
     if check and ret.returncode != 0:
         raise RuntimeError(f"failed to execute {cmd_pretty}")
@@ -63,8 +77,44 @@ def disable_output_buffer():
 
 
 def prefix_idx(prefix: str, seq: list[str]) -> int:
+    """
+    >>> prefix_idx("a", ["c", "a", "d"])
+    1
+    >>> prefix_idx("a", ["c", "d", "e", "a"])
+    3
+    """
     for idx, elem in enumerate(seq):
         if re.match(rf"{prefix}\b", elem):
             return idx
 
     raise ValueError(f"prefix {prefix!r} not found in sequence")
+
+
+T = TypeVar("T")
+
+
+def list_strip_head(data: list[T], head: list[T]) -> list[T]:
+    """
+    >>> list_strip_head([1,2,3], [])
+    [1, 2, 3]
+    >>> list_strip_head([1,2,3], [1,2])
+    [3]
+    >>> list_strip_head([1,2,3], [1,2,3])
+    []
+    >>> list_strip_head([1,2,3,4,5], [1,2])
+    [3, 4, 5]
+    """
+    idx = 0
+    for elem_a, elem_b in zip(data, head):
+        if elem_a == elem_b:
+            idx += 1
+        else:
+            break
+
+    return data[idx:]
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()
diff --git a/src/debmagic/v0.py b/src/debmagic/v0.py
@@ -1,5 +1,5 @@
 from ._build import Build
-from ._modules import autotools, dh
+from ._module import autotools, dh
 from ._package import package
 from ._preset import Preset
 from ._utils import run_cmd
diff --git a/tests/integration/packages/.gitignore b/tests/integration/packages/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!.ignore
diff --git a/tests/integration/packages/.ignore b/tests/integration/packages/.ignore
@@ -0,0 +1 @@
+*
diff --git a/tests/integration/packages/htop/rules.py b/tests/integration/packages/htop/rules.py
@@ -29,7 +29,7 @@ def configure(build: Build):
     autotools_mod.autoreconf(build)
     autotools.configure(
         build,
-        ["--enable-openvz", "--enable-vserver", "--enable-unicode"] + configure_params,
+        ["--enable-openvz", "--enable-vserver", "--enable-unicode", *configure_params],
     )
 
 
diff --git a/tests/integration/test_packages.py b/tests/integration/test_packages.py
@@ -33,7 +33,7 @@ def fetch_sources(package_name: str, version: str) -> Path:
 ]
 
 
-@pytest.mark.parametrize("package, version", package_fixtures, ids=map(lambda x: x[0], package_fixtures))
+@pytest.mark.parametrize("package, version", package_fixtures, ids=[x[0] for x in package_fixtures])
 def test_build_package(package: str, version: str):
     # TODO use sandbox/container, lxd?
     repo_dir = fetch_sources(package_name=package, version=version)
