diff --git a/pyproject.toml b/pyproject.toml index 33dea0b..56ed6e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,11 +31,14 @@ exclude_gitignore = true [tool.ruff] line-length = 120 target-version = "py312" -exclude = [".git", ".idea", ".mypy_cache", ".venv*", "docs", "debian"] +extend-exclude = [".idea", ".mypy_cache", ".venv*", "docs", "debian", "__pycache__", "*.egg_info"] [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "I", "PLE", "PLW"] -ignore = ["E722"] +select = ["E", "W", "F", "I", "C", "N", "PL", "RUF", "I001"] +ignore = ["E722", "PLR2004", "PLR0912", "PLR5501", "PLC0415"] +mccabe.max-complexity = 25 +pylint.max-args = 10 [tool.pytest] testpaths = ["tests"] +addopts = ["--ignore=tests/integration/packages"] diff --git a/src/debmagic/_build.py b/src/debmagic/_build.py index 02938ea..9c64917 100644 --- a/src/debmagic/_build.py +++ b/src/debmagic/_build.py @@ -1,9 +1,9 @@ from __future__ import annotations -import shlex import typing from dataclasses import dataclass, field from pathlib import Path +from typing import Sequence from ._build_stage import BuildStage from ._utils import run_cmd @@ -29,21 +29,13 @@ class Build: dry_run: bool = False _completed_stages: set[BuildStage] = field(default_factory=set) - _selected_packages: list[BinaryPackage] | None = field(default_factory=list) - def cmd(self, cmd: list[str] | str, **kwargs): + def cmd(self, cmd: Sequence[str] | str, **kwargs): """ execute a command, auto-converts command strings/lists. use this to supports build dry-runs. """ - cmd_args: list[str] | str = cmd - is_shell = kwargs.get("shell") - if not is_shell and isinstance(cmd, str): - cmd_args = shlex.split(cmd) - elif is_shell and not isinstance(cmd, str): - cmd_args = shlex.join(cmd) - - run_cmd(cmd_args, dry_run=self.dry_run, **kwargs) + run_cmd(cmd, dry_run=self.dry_run, **kwargs) @property def install_dirs(self) -> dict[str, Path]: @@ -52,17 +44,20 @@ def install_dirs(self) -> dict[str, Path]: def select_packages(self, names: set[str]): """only build those packages""" - if not names: - self._selected_packages = None + self.binary_packages = [] - self._selected_packages = list() - for pkg in self.binary_packages: + for pkg in self.source_package.binary_packages: if pkg.name in names: - self._selected_packages.append(pkg) + self.binary_packages.append(pkg) def filter_packages(self, package_filter: PackageFilter) -> None: """apply filter to only build those packages""" - self.select_packages({pkg.name for pkg in package_filter.get_packages(self.binary_packages)}) + self.select_packages({pkg.name for pkg in package_filter.get_packages(self.source_package.binary_packages)}) + + def filtered_binary_packages(self, names: set[str]) -> typing.Iterator[BinaryPackage]: + for pkg in self.binary_packages: + if pkg.name in names: + yield pkg def is_stage_completed(self, stage: BuildStage) -> bool: return stage in self._completed_stages @@ -95,13 +90,12 @@ def run( for preset in self.presets: print(f"debmagic: trying preset {preset}...") if preset_stage_function := preset.get_stage(stage): - print("debmagic: preset has function") + print("debmagic: running stage from preset") preset_stage_function(self) self._mark_stage_done(stage) break # stop preset processing if not self.is_stage_completed(stage): - breakpoint() raise RuntimeError(f"{stage!s} stage was never executed") if stage == target_stage: diff --git a/src/debmagic/_modules/autodetect.py b/src/debmagic/_module/autodetect.py similarity index 100% rename from src/debmagic/_modules/autodetect.py rename to src/debmagic/_module/autodetect.py diff --git a/src/debmagic/_modules/autotools.py b/src/debmagic/_module/autotools.py similarity index 92% rename from src/debmagic/_modules/autotools.py rename to src/debmagic/_module/autotools.py index bc5b901..fc8613a 100644 --- a/src/debmagic/_modules/autotools.py +++ b/src/debmagic/_module/autotools.py @@ -47,16 +47,16 @@ def configure(self, build: Build, args: list[str] | None = None): # TODO: show some config.log if configure failed def build(self, build: Build, args: list[str] = []): - args = [f"-j{build.parallel}"] + args + args = [f"-j{build.parallel}", *args] - build.cmd(["make"] + args, cwd=build.source_dir) + build.cmd(["make", *args], cwd=build.source_dir) # TODO: def test(): run make test or make check def install(self, build: Build, args: list[str] = []): # TODO: figure out installdir handling for multi package builds destdir = build.install_dirs[build.source_package.name] - build.cmd(["make", f"DESTDIR={destdir}", "install"] + args, cwd=build.source_dir) + build.cmd(["make", f"DESTDIR={destdir}", "install", *args], cwd=build.source_dir) def autoreconf(build: Build): diff --git a/src/debmagic/_modules/default.py b/src/debmagic/_module/default.py similarity index 100% rename from src/debmagic/_modules/default.py rename to src/debmagic/_module/default.py diff --git a/src/debmagic/_modules/dh.py b/src/debmagic/_module/dh.py similarity index 80% rename from src/debmagic/_modules/dh.py rename to src/debmagic/_module/dh.py index 4d36c62..bf58875 100644 --- a/src/debmagic/_modules/dh.py +++ b/src/debmagic/_module/dh.py @@ -12,7 +12,7 @@ from .._build import Build from .._package import SourcePackage from .._preset import Preset as PresetBase -from .._utils import prefix_idx, run_cmd +from .._utils import list_strip_head, prefix_idx, run_cmd class DHSequenceID(StrEnum): @@ -20,6 +20,9 @@ class DHSequenceID(StrEnum): build = "build" build_arch = "build-arch" build_indep = "build-indep" + install = "install" + install_arch = "install-arch" + install_indep = "install-indep" binary = "binary" binary_arch = "binary-arch" binary_indep = "binary-indep" @@ -33,16 +36,16 @@ def __init__(self, dh_invocation: str | None = None): if not dh_invocation: dh_invocation = "dh" self._dh_invocation: str = dh_invocation - self._overrides: dict[str, DHOverride] = dict() + self._overrides: dict[str, DHOverride] = {} self._initialized = False # debmagic's stages, with matching commands from the dh sequence - self._clean_seq: list[str] = list() - self._configure_seq: list[str] = list() - self._build_seq: list[str] = list() - self._test_seq: list[str] = list() - self._install_seq: list[str] = list() - self._package_seq: list[str] = list() + self._clean_seq: list[str] = [] + self._configure_seq: list[str] = [] + self._build_seq: list[str] = [] + self._test_seq: list[str] = [] + self._install_seq: list[str] = [] + self._package_seq: list[str] = [] # all seen sequence cmd ids (the dh command script itself) self._seq_ids: set[str] = set() @@ -106,10 +109,10 @@ def _populate_stages(self, dh_invocation: str, base_dir: Path) -> None: self._clean_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.clean) ## untangle "build" to configure & build & test - build_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.build) - if build_seq[-1] != "create-stamp debian/debhelper-build-stamp": + build_seq_raw = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.build) + if build_seq_raw[-1] != "create-stamp debian/debhelper-build-stamp": raise RuntimeError("build stamp creation line missing from dh build sequence") - build_seq = build_seq[:-1] # remove that stamp line + build_seq = build_seq_raw[:-1] # remove that stamp line auto_cfg_idx = prefix_idx("dh_auto_configure", build_seq) # up to including dh_auto_configure @@ -125,14 +128,12 @@ def _populate_stages(self, dh_invocation: str, base_dir: Path) -> None: self._test_seq = build_seq[auto_test_idx:] ## untangle "binary" to install & package + install_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.install) + self._install_seq = list_strip_head(install_seq, build_seq_raw) binary_seq = self._get_dh_seq(base_dir, dh_invocation, DHSequenceID.binary) - build_stamp_idx = prefix_idx("create-stamp debian/debhelper-build-stamp", binary_seq) - auto_install_idx = prefix_idx("dh_auto_install", binary_seq) - # one after the build-stamp, up to including dh_auto_install - self._install_seq = binary_seq[build_stamp_idx + 1 : auto_install_idx + 1] - # assume everything else is packing (which is a bit wrong, but packing & installing is mixed in dh) - self._package_seq = binary_seq[auto_install_idx + 1 :] + self._package_seq = list_strip_head(binary_seq, install_seq) + # register all sequence items for validity checks for seq in ( self._clean_seq, self._configure_seq, @@ -148,7 +149,7 @@ def _populate_stages(self, dh_invocation: str, base_dir: Path) -> None: def _get_dh_seq(self, base_dir: Path, dh_invocation: str, seq: DHSequenceID) -> list[str]: dh_base_cmd = shlex.split(dh_invocation) - cmd = dh_base_cmd + [str(seq), "--no-act"] + cmd = [*dh_base_cmd, str(seq), "--no-act"] proc = run_cmd(cmd, cwd=base_dir, capture_output=True, text=True) lines = proc.stdout.splitlines() return [line.strip() for line in lines] diff --git a/src/debmagic/_modules/python.py b/src/debmagic/_module/python.py similarity index 100% rename from src/debmagic/_modules/python.py rename to src/debmagic/_module/python.py diff --git a/src/debmagic/_package.py b/src/debmagic/_package.py index 650c104..3c4e20e 100644 --- a/src/debmagic/_package.py +++ b/src/debmagic/_package.py @@ -95,7 +95,7 @@ class PackageFilter(Flag): architecture_independent = auto() def get_packages(self, binary_packages: list[BinaryPackage]) -> list[BinaryPackage]: - ret: list[BinaryPackage] = list() + ret: list[BinaryPackage] = [] for pkg in binary_packages: if self.architecture_independent and pkg.arch_dependent: @@ -157,7 +157,7 @@ def something(arg: str = 'stuff'): # find arguments and its types to guess argparsing args_raw = inspect.getfullargspec(func) - args: CustomFuncArgsT = dict() + args: CustomFuncArgsT = {} default_count = 0 if not args_raw.defaults else len(args_raw.defaults) default_start = len(args_raw.args) - default_count for idx, arg in enumerate(args_raw.args): @@ -253,7 +253,7 @@ def package( presets: list[Preset] = as_presets(preset) # apply default preset last - from ._modules.default import Preset as DefaultPreset + from ._module.default import Preset as DefaultPreset presets.append(DefaultPreset()) @@ -263,7 +263,7 @@ def package( src_pkg: SourcePackage | None = None # which binary packages should be produced? - bin_pkgs: list[BinaryPackage] = list() + bin_pkgs: list[BinaryPackage] = [] for block in deb822.DebControl.iter_paragraphs( (rules_file.package_dir / "debian/control").open(), diff --git a/src/debmagic/_preset.py b/src/debmagic/_preset.py index 3eb0eea..a10a427 100644 --- a/src/debmagic/_preset.py +++ b/src/debmagic/_preset.py @@ -96,7 +96,7 @@ def _get_member(obj: T | type[T], stage: BuildStage) -> BuildStep | _PresetBuild def as_presets(preset_elements: PresetsT) -> list[Preset]: - presets: list[Preset] = list() + presets: list[Preset] = [] if isinstance(preset_elements, list): for preset_module in preset_elements: diff --git a/src/debmagic/_rules_file.py b/src/debmagic/_rules_file.py index 6f1ce5d..eff1414 100644 --- a/src/debmagic/_rules_file.py +++ b/src/debmagic/_rules_file.py @@ -13,7 +13,7 @@ def find_rules_file() -> RulesFile: for frame in inspect.stack(): file_path = Path(frame.filename) # TODO further validation, be in debian/ - if file_path.name == "rules" or file_path.name == "rules.py": + if file_path.name in {"rules", "rules.py"}: return RulesFile( package_dir=file_path.parent.parent.resolve(), local_vars=frame.frame.f_locals, diff --git a/src/debmagic/_utils.py b/src/debmagic/_utils.py index 582ac3d..680b696 100644 --- a/src/debmagic/_utils.py +++ b/src/debmagic/_utils.py @@ -3,6 +3,7 @@ import shlex import subprocess import sys +from typing import Sequence, TypeVar class Namespace: @@ -11,8 +12,8 @@ class Namespace: """ def __init__(self, **kwargs): - for name in kwargs: - setattr(self, name, kwargs[name]) + for name, value in kwargs.items(): + setattr(self, name, value) def __eq__(self, other): if not isinstance(other, Namespace): @@ -35,20 +36,33 @@ def get(self, key): return self.__dict__.get(key) -def run_cmd(args: list[str] | str, check: bool = True, dry_run: bool = False, **kwargs) -> subprocess.CompletedProcess: - shell = kwargs.get("shell") - if shell: - print(f"debmagic: {args}") +def run_cmd( + cmd: Sequence[str] | str, + check: bool = True, + dry_run: bool = False, + **kwargs, +) -> subprocess.CompletedProcess: + cmd_args: Sequence[str] | str = cmd + cmd_pretty: str + + if kwargs.get("shell"): + if not isinstance(cmd, str): + cmd_args = shlex.join(cmd) + else: + if isinstance(cmd, str): + cmd_args = shlex.split(cmd) + + if isinstance(cmd, str): + cmd_pretty = cmd else: - if not isinstance(args, (list, tuple)): - raise ValueError("need list/tuple as command arguments when not using shell=True") - cmd_pretty = shlex.join(args) - print(f"debmagic: {cmd_pretty}") + cmd_pretty = shlex.join(cmd_args) + + print(f"debmagic: {cmd_pretty}") if dry_run: - return subprocess.CompletedProcess(args, 0) + return subprocess.CompletedProcess(cmd_args, 0) - ret = subprocess.run(args, check=False, **kwargs) + ret = subprocess.run(cmd_args, check=False, **kwargs) if check and ret.returncode != 0: raise RuntimeError(f"failed to execute {cmd_pretty}") @@ -63,8 +77,44 @@ def disable_output_buffer(): def prefix_idx(prefix: str, seq: list[str]) -> int: + """ + >>> prefix_idx("a", ["c", "a", "d"]) + 1 + >>> prefix_idx("a", ["c", "d", "e", "a"]) + 3 + """ for idx, elem in enumerate(seq): if re.match(rf"{prefix}\b", elem): return idx raise ValueError(f"prefix {prefix!r} not found in sequence") + + +T = TypeVar("T") + + +def list_strip_head(data: list[T], head: list[T]) -> list[T]: + """ + >>> list_strip_head([1,2,3], []) + [1, 2, 3] + >>> list_strip_head([1,2,3], [1,2]) + [3] + >>> list_strip_head([1,2,3], [1,2,3]) + [] + >>> list_strip_head([1,2,3,4,5], [1,2]) + [3, 4, 5] + """ + idx = 0 + for elem_a, elem_b in zip(data, head): + if elem_a == elem_b: + idx += 1 + else: + break + + return data[idx:] + + +if __name__ == "__main__": + import doctest + + doctest.testmod() diff --git a/src/debmagic/v0.py b/src/debmagic/v0.py index 9deb090..17017f5 100644 --- a/src/debmagic/v0.py +++ b/src/debmagic/v0.py @@ -1,5 +1,5 @@ from ._build import Build -from ._modules import autotools, dh +from ._module import autotools, dh from ._package import package from ._preset import Preset from ._utils import run_cmd diff --git a/tests/integration/packages/.gitignore b/tests/integration/packages/.gitignore new file mode 100644 index 0000000..47c234d --- /dev/null +++ b/tests/integration/packages/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!.ignore diff --git a/tests/integration/packages/.ignore b/tests/integration/packages/.ignore new file mode 100644 index 0000000..72e8ffc --- /dev/null +++ b/tests/integration/packages/.ignore @@ -0,0 +1 @@ +* diff --git a/tests/integration/packages/htop/rules.py b/tests/integration/packages/htop/rules.py index 870efb5..b8ac4c0 100755 --- a/tests/integration/packages/htop/rules.py +++ b/tests/integration/packages/htop/rules.py @@ -29,7 +29,7 @@ def configure(build: Build): autotools_mod.autoreconf(build) autotools.configure( build, - ["--enable-openvz", "--enable-vserver", "--enable-unicode"] + configure_params, + ["--enable-openvz", "--enable-vserver", "--enable-unicode", *configure_params], ) diff --git a/tests/integration/test_packages.py b/tests/integration/test_packages.py index 3dc6291..9b4cdc3 100644 --- a/tests/integration/test_packages.py +++ b/tests/integration/test_packages.py @@ -33,7 +33,7 @@ def fetch_sources(package_name: str, version: str) -> Path: ] -@pytest.mark.parametrize("package, version", package_fixtures, ids=map(lambda x: x[0], package_fixtures)) +@pytest.mark.parametrize("package, version", package_fixtures, ids=[x[0] for x in package_fixtures]) def test_build_package(package: str, version: str): # TODO use sandbox/container, lxd? repo_dir = fetch_sources(package_name=package, version=version)