Merge pull request #11 from SFTtech/milo/config-loading

feat(debmagic): add config file parsing with overrides from cli args

Author: mikonse

.github/workflows/ci.yml

@@ -108,7 +108,6 @@ jobs:
   #     matrix:
   #       python-version:
   #       - "3.12"
-  #       - "3.13"
   #       os: [ubuntu-latest]
   #   steps:
   #     - uses: actions/checkout@v5

debian/control

@@ -46,6 +46,7 @@ Architecture: all
 Depends:
  python3-debmagic-common,
  python3-pydantic,
+ python3-pydantic-settings,
  ${misc:Depends},
  ${python3:Depends}
 Multi-Arch: foreign

packages/debmagic/pyproject.toml

@@ -7,7 +7,11 @@ license = "GPL-2.0-or-later"
 # readme = "../../README.md"
 requires-python = ">=3.12"
 classifiers = ["Programming Language :: Python :: 3"]
-dependencies = ["debmagic-common", "pydantic>=2,<3"]
+dependencies = [
+    "debmagic-common",
+    "pydantic>=2,<3",
+    "pydantic-settings>=2.12.0",
+]
 
 [project.urls]
 homepage = "https://github.com/SFTtech/debmagic"

packages/debmagic/src/debmagic/cli/_build_driver/build.py

@@ -6,19 +6,20 @@
 
 from .._config import DebmagicConfig
 from .common import BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, PackageDescription
+from .config import BuildDriverConfig
 from .driver_bare import BuildDriverBare
 from .driver_docker import BuildDriverDocker
 from .driver_lxd import BuildDriverLxd
 
 
-def _create_driver(build_driver: BuildDriverType, config: BuildConfig, additional_args: list[str]) -> BuildDriver:
+def _create_driver(build_driver: BuildDriverType, build_config: BuildConfig, config: BuildDriverConfig) -> BuildDriver:
     match build_driver:
         case "docker":
-            return BuildDriverDocker.create(config=config, additional_args=additional_args)
+            return BuildDriverDocker.create(config=build_config, driver_config=config.docker)
         case "lxd":
-            return BuildDriverLxd.create(config=config, additional_args=additional_args)
+            return BuildDriverLxd.create(config=build_config, driver_config=config.lxd)
         case "bare":
-            return BuildDriverBare.create(config=config, additional_args=additional_args)
+            return BuildDriverBare.create(config=build_config, driver_config=config.bare)
 
 
 def _driver_from_build_root(build_root: Path):
@@ -68,9 +69,7 @@ def _get_package_build_root_and_identifier(config: DebmagicConfig, package: Pack
     return package_identifier, build_root
 
 
-def _prepare_build_env(
-    config: DebmagicConfig, package: PackageDescription, output_dir: Path, dry_run: bool
-) -> BuildConfig:
+def _prepare_build_env(config: DebmagicConfig, package: PackageDescription, output_dir: Path) -> BuildConfig:
     package_identifier, build_root = _get_package_build_root_and_identifier(config, package)
     if build_root.exists():
         shutil.rmtree(build_root)
@@ -82,7 +81,7 @@ def _prepare_build_env(
         build_root_dir=build_root,
         distro="debian",
         distro_version="trixie",
-        dry_run=dry_run,
+        dry_run=config.dry_run,
         sign_package=False,
     )
 
@@ -107,12 +106,10 @@ def build(
     package: PackageDescription,
     build_driver: BuildDriverType,
     output_dir: Path,
-    additional_args: list[str],
-    dry_run: bool = False,
 ):
-    build_config = _prepare_build_env(config=config, package=package, output_dir=output_dir, dry_run=dry_run)
+    build_config = _prepare_build_env(config=config, package=package, output_dir=output_dir)
 
-    driver = _create_driver(build_driver, build_config, additional_args)
+    driver = _create_driver(build_driver, build_config, config.driver_config)
     _write_build_metadata(build_config, driver)
     try:
         driver.run_command(["apt-get", "-y", "build-dep", "."], cwd=build_config.build_source_dir, requires_root=True)

packages/debmagic/src/debmagic/cli/_build_driver/common.py

@@ -1,7 +1,7 @@
 import abc
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Literal, Self, Sequence
+from typing import Literal, Self, Sequence, TypeVar
 
 from pydantic import BaseModel
 
@@ -67,10 +67,13 @@ def create_dirs(self):
         self.build_source_dir.mkdir(exist_ok=True, parents=True)
 
 
-class BuildDriver:
+ConfigT = TypeVar("ConfigT")
+
+
+class BuildDriver[ConfigT]:
     @classmethod
     @abc.abstractmethod
-    def create(cls, config: BuildConfig, additional_args: list[str]) -> Self:
+    def create(cls, config: BuildConfig, driver_config: ConfigT) -> Self:
         pass
 
     @classmethod

packages/debmagic/src/debmagic/cli/_build_driver/config.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+
+from .driver_bare import BareDriverConfig
+from .driver_docker import DockerDriverConfig
+from .driver_lxd import LxdDriverConfig
+
+
+class BuildDriverConfig(BaseModel):
+    bare: BareDriverConfig = BareDriverConfig()
+    docker: DockerDriverConfig = DockerDriverConfig()
+    lxd: LxdDriverConfig = LxdDriverConfig()

packages/debmagic/src/debmagic/cli/_build_driver/driver_bare.py

@@ -3,16 +3,21 @@
 from typing import Sequence
 
 from debmagic.common.utils import run_cmd, run_cmd_in_foreground
+from pydantic import BaseModel
 
 from .common import BuildConfig, BuildDriver
 
 
-class BuildDriverBare(BuildDriver):
+class BareDriverConfig(BaseModel):
+    pass
+
+
+class BuildDriverBare(BuildDriver[BareDriverConfig]):
     def __init__(self, config: BuildConfig) -> None:
         self._config = config
 
     @classmethod
-    def create(cls, config: BuildConfig, additional_args: list[str]):
+    def create(cls, config: BuildConfig, driver_config: BareDriverConfig):
         return cls(config=config)
 
     def run_command(self, cmd: Sequence[str | Path], cwd: Path | None = None, requires_root: bool = False):

packages/debmagic/src/debmagic/cli/_build_driver/driver_docker.py

@@ -1,10 +1,10 @@
-import argparse
+import os
 import uuid
 from pathlib import Path
 from typing import Self, Sequence
 
 from debmagic.common.utils import run_cmd, run_cmd_in_foreground
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from .common import (
     BuildConfig,
@@ -17,21 +17,42 @@
 
 BUILD_DIR_IN_CONTAINER = Path("/debmagic")
 
+DOCKER_USER = "user"
+
 DOCKERFILE_TEMPLATE = f"""
 FROM {{base_image}}
 
-RUN apt-get update && apt-get -y install dpkg-dev python3
+ARG USERNAME={DOCKER_USER}
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+RUN apt-get update && apt-get install -y sudo dpkg-dev python3
+
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
+    && echo $USERNAME ALL=\\(root\\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME
 
 RUN mkdir -p {BUILD_DIR_IN_CONTAINER}
+RUN chown $USERNAME:$USERNAME {BUILD_DIR_IN_CONTAINER}
+USER $USERNAME
 ENTRYPOINT ["sleep", "infinity"]
 """
 
 
+class DockerDriverConfig(BaseModel):
+    base_image: str | None = Field(
+        default=None,
+        description="Can be used to override the base image which the docker driver "
+        "uses to create the build environment",
+    )
+
+
 class DockerDriverBuildMetadata(BaseModel):
     container_name: str
 
 
-class BuildDriverDocker(BuildDriver):
+class BuildDriverDocker(BuildDriver[DockerDriverConfig]):
     def __init__(self, build_root: Path, dry_run: bool, container_name: str):
         self._build_root = build_root
         self._dry_run = dry_run
@@ -44,30 +65,28 @@ def _translate_path_in_container(self, path_in_source: Path) -> Path:
         rel = path_in_source.relative_to(self._build_root)
         return BUILD_DIR_IN_CONTAINER / rel
 
-    @staticmethod
-    def _parse_args(args: list[str]) -> argparse.Namespace:
-        parser = argparse.ArgumentParser()
-        parser.add_argument("--docker-image", type=str)
-        return parser.parse_args(args)
-
     @classmethod
-    def create(cls, config: BuildConfig, additional_args: list[str]) -> Self:
-        args = cls._parse_args(additional_args)
-
-        base_image = f"docker.io/{config.distro}:{config.distro_version}"
-        if args.docker_image is not None:
-            base_image = args.docker_image
+    def create(cls, config: BuildConfig, driver_config: DockerDriverConfig) -> Self:
+        base_image = driver_config.base_image or f"docker.io/{config.distro}:{config.distro_version}"
 
         formatted_dockerfile = DOCKERFILE_TEMPLATE.format(base_image=base_image)
 
         dockerfile_path = config.build_temp_dir / "Dockerfile"
         dockerfile_path.write_text(formatted_dockerfile)
 
         docker_image_name = f"debmagic-{config.build_identifier}"
+
+        additional_args = []
+        if not os.getuid() == 0:
+            # to reduce potential permission problems with missing user remappings on some systems
+            # we simply create the build user inside the docker container with the same uid / gid as our host user
+            additional_args.extend(["--build-arg", f"USER_UID={os.getuid()}", "--build-arg", f"USER_GID={os.getgid()}"])
+
         ret = run_cmd(
             [
                 "docker",
                 "build",
+                *additional_args,
                 "--tag",
                 docker_image_name,
                 "-f",
@@ -112,15 +131,16 @@ def get_build_metadata(self) -> DriverSpecificBuildMetadata:
         return meta.model_dump()
 
     def run_command(self, cmd: Sequence[str | Path], cwd: Path | None = None, requires_root: bool = False):
-        del requires_root  # we assume to always be root in the container
+        conditional_args: list[str | Path] = []
 
         if cwd:
             cwd = self._translate_path_in_container(cwd)
-            cwd_args: list[str | Path] = ["--workdir", cwd]
-        else:
-            cwd_args = []
+            conditional_args.extend(["--workdir", cwd])
+
+        if requires_root:
+            conditional_args.extend(["--user", "root"])
 
-        ret = run_cmd(["docker", "exec", *cwd_args, self._container_name, *cmd], dry_run=self._dry_run)
+        ret = run_cmd(["docker", "exec", *conditional_args, self._container_name, *cmd], dry_run=self._dry_run)
         if ret.returncode != 0:
             raise BuildError("Error building package")
 

packages/debmagic/src/debmagic/cli/_build_driver/driver_lxd.py

@@ -1,12 +1,18 @@
 from pathlib import Path
 from typing import Self, Sequence
 
+from pydantic import BaseModel
+
 from .common import BuildConfig, BuildDriver
 
 
-class BuildDriverLxd(BuildDriver):
+class LxdDriverConfig(BaseModel):
+    pass
+
+
+class BuildDriverLxd(BuildDriver[LxdDriverConfig]):
     @classmethod
-    def create(cls, config: BuildConfig, additional_args: list[str]) -> Self:
+    def create(cls, config: BuildConfig, driver_config: LxdDriverConfig) -> Self:
         raise NotImplementedError()
 
     @classmethod

packages/debmagic/src/debmagic/cli/_config.py

@@ -1,20 +1,63 @@
-import tomllib
+import argparse
 from pathlib import Path
-from typing import Sequence
+from typing import Annotated, ClassVar
 
-from pydantic import BaseModel
+from pydantic import AfterValidator, Field
+from pydantic_settings import (
+    BaseSettings,
+    CliApp,
+    CliSettingsSource,
+    PydanticBaseSettingsSource,
+    TomlConfigSettingsSource,
+)
 
+from ._build_driver.config import BuildDriverConfig
 
-class DebmagicConfig(BaseModel):
-    # driver: BuildDriverType | None = None
-    temp_build_dir: Path = Path("/tmp/debmagic")
 
+def resolve_path(p: Path) -> Path:
+    return p.resolve()
 
-def merge_configs(configs: Sequence[DebmagicConfig]) -> DebmagicConfig:
-    return configs[-1]
 
+class DebmagicConfig(BaseSettings, cli_kebab_case=True, cli_avoid_json=True):
+    _config_file_paths: ClassVar[list[Path]] = []
 
-def load_config(config_path: Path) -> DebmagicConfig:
-    with config_path.open("rb") as f:
-        content = tomllib.load(f)
-    return DebmagicConfig.model_validate(content)
+    driver_config: BuildDriverConfig = BuildDriverConfig()
+
+    temp_build_dir: Annotated[
+        Path,
+        Field(description="Temporary directory on the local machine used as root directory for all package builds"),
+        AfterValidator(resolve_path),
+    ] = Path("/tmp/debmagic")
+
+    dry_run: Annotated[bool, Field(description="don't actually run anything that changes the system/package state")] = (
+        False
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        # FIXME: once pydantic properly supports dynamic runtime paths for these kinds of use cases
+        # use a more sane method of injecting additional paths to load
+        # see https://github.com/pydantic/pydantic-settings/issues/259
+        return tuple(TomlConfigSettingsSource(settings_cls, p) for p in cls._config_file_paths)
+
+
+def get_config_argparser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(add_help=False)
+    CliSettingsSource(DebmagicConfig, root_parser=parser)
+    return parser
+
+
+def load_config(args: argparse.Namespace, config_paths: list[Path]) -> DebmagicConfig:
+    DebmagicConfig._config_file_paths = config_paths
+    # parser = argparse.ArgumentParser(add_help=False)
+    source = CliSettingsSource(DebmagicConfig)
+    config = CliApp.run(DebmagicConfig, cli_args=args, cli_settings_source=source)
+
+    return config