Merge pull request ceph#65936 from tchaikov/wip-build-cephadm-with-deb

cephadm, debian/rules: Use system packages for cephadm bundled dependencies

Reviewed-by: John Mulligan <[email protected]>

Author: tchaikov

debian/control

@@ -107,6 +107,8 @@ Build-Depends: automake,
                python3-requests <pkg.ceph.check>,
                python3-scipy <pkg.ceph.check>,
                python3-onelogin-saml2 <pkg.ceph.check>,
+               python3-jinja2,
+               python3-markupsafe,
                python3-setuptools,
                python3-sphinx,
                python3-venv,

debian/rules

@@ -28,6 +28,7 @@ extraopts += -DWITH_CEPHFS_JAVA=ON
 extraopts += -DWITH_CEPHFS_SHELL=ON
 extraopts += -DWITH_SYSTEMD=ON -DCEPH_SYSTEMD_ENV_DIR=/etc/default
 extraopts += -DWITH_GRAFANA=ON
+extraopts += -DCEPHADM_BUNDLED_DEPENDENCIES=deb
 ifeq ($(DEB_HOST_ARCH), amd64)
   extraopts += -DWITH_RBD_RWL=ON
 else

src/cephadm/build.py

@@ -154,6 +154,7 @@ def enabled(self):
 class DependencyMode(enum.Enum):
     pip = enum.auto()
     rpm = enum.auto()
+    deb = enum.auto()
     none = enum.auto()
 
 
@@ -169,6 +170,8 @@ def __init__(self, cli_args):
             self._setup_pip()
         elif self.deps_mode == DependencyMode.rpm:
             self._setup_rpm()
+        elif self.deps_mode == DependencyMode.deb:
+            self._setup_deb()
 
     def _setup_pip(self):
         if self._maj_min == (3, 6):
@@ -180,6 +183,9 @@ def _setup_pip(self):
     def _setup_rpm(self):
         self.requirements = [InstallSpec(**v) for v in PY_REQUIREMENTS]
 
+    def _setup_deb(self):
+        self.requirements = [InstallSpec(**v) for v in PY_REQUIREMENTS]
+
 
 class DependencyInfo:
     """Type for tracking bundled dependencies."""
@@ -336,7 +342,9 @@ def _install_deps(tempdir, config):
         return _install_pip_deps(tempdir, config)
     if config.deps_mode == DependencyMode.rpm:
         return _install_rpm_deps(tempdir, config)
-    raise ValueError(f'unexpected deps mode: {deps.mode}')
+    if config.deps_mode == DependencyMode.deb:
+        return _install_deb_deps(tempdir, config)
+    raise ValueError(f'unexpected deps mode: {config.deps_mode}')
 
 
 def _install_pip_deps(tempdir, config):
@@ -437,7 +445,26 @@ def _install_rpm_deps(tempdir, config):
     return dinfo
 
 
-def _gather_rpm_package_dirs(paths):
+def _install_deb_deps(tempdir, config):
+    log.info("Installing dependencies using Debian packages")
+    dinfo = DependencyInfo(config)
+    for pkg in config.requirements:
+        log.info(f"Looking for debian package for: {pkg.name!r}")
+        _deps_from_deb(tempdir, config, dinfo, pkg.name)
+    return dinfo
+
+
+def _gather_package_dirs(paths, expected_parent_dir):
+    """Parse package file listing to find Python package directories.
+
+    Args:
+        paths: List of file paths from package listing
+        expected_parent_dir: Expected parent directory name (e.g., 'site-packages' for RPM,
+                           'dist-packages' for Debian)
+
+    Returns:
+        Tuple of (metadata_dir, package_dirs)
+    """
     # = The easy way =
     # the top_level.txt file can be used to determine where the python packages
     # actually are. We need all of those and the meta-data dir (parent of
@@ -456,7 +483,7 @@ def _gather_rpm_package_dirs(paths):
     # = The hard way =
     # loop through the directories to find the .dist-info dir (containing the
     # mandatory METADATA file, according to the spec) and once we know the
-    # location of dist info we find the sibling paths from the rpm listing
+    # location of dist info we find the sibling paths from the package listing
     dist_info = None
     ppaths = []
     for path in paths:
@@ -467,9 +494,9 @@ def _gather_rpm_package_dirs(paths):
             break
     if not dist_info:
         raise ValueError('no .dist-info METADATA found')
-    if not dist_info.parent.name == 'site-packages':
+    if dist_info.parent.name != expected_parent_dir:
         raise ValueError(
-            'unexpected parent directory (not site-packages):'
+            f'unexpected parent directory (not {expected_parent_dir}):'
             f' {dist_info.parent.name}'
         )
     siblings = [
@@ -478,6 +505,31 @@ def _gather_rpm_package_dirs(paths):
     return dist_info, siblings
 
 
+def _copy_package_files(tempdir, paths, expected_parent_dir):
+    """Copy package files to the build directory.
+
+    Args:
+        tempdir: Temporary directory to copy files to
+        paths: List of file paths from package listing
+        expected_parent_dir: Expected parent directory name per packaging convention:
+            - 'site-packages' for RPM-based distributions
+            - 'dist-packages' for Debian-based distributions
+
+    Returns:
+        None
+    """
+    meta_dir, pkg_dirs = _gather_package_dirs(paths, expected_parent_dir)
+    meta_dest = tempdir / meta_dir.name
+    log.info(f"Copying {meta_dir} to {meta_dest}")
+    # copy the meta data directory
+    shutil.copytree(meta_dir, meta_dest, ignore=_ignore_cephadmlib)
+    # copy all the package directories
+    for pkg_dir in pkg_dirs:
+        pkg_dest = tempdir / pkg_dir.name
+        log.info(f"Copying {pkg_dir} to {pkg_dest}")
+        shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib)
+
+
 def _deps_from_rpm(tempdir, config, dinfo, pkg):
     # first, figure out what rpm provides a particular python lib
     dist = f'python3.{sys.version_info.minor}dist({pkg})'.lower()
@@ -513,16 +565,92 @@ def _deps_from_rpm(tempdir, config, dinfo, pkg):
         ['rpm', '-ql', rpmname], check=True, stdout=subprocess.PIPE
     )
     paths = [l.decode('utf8') for l in res.stdout.splitlines()]
-    meta_dir, pkg_dirs = _gather_rpm_package_dirs(paths)
-    meta_dest = tempdir / meta_dir.name
-    log.info(f"Copying {meta_dir} to {meta_dest}")
-    # copy the meta data directory
-    shutil.copytree(meta_dir, meta_dest, ignore=_ignore_cephadmlib)
-    # copy all the package directories
-    for pkg_dir in pkg_dirs:
-        pkg_dest = tempdir / pkg_dir.name
-        log.info(f"Copying {pkg_dir} to {pkg_dest}")
-        shutil.copytree(pkg_dir, pkg_dest, ignore=_ignore_cephadmlib)
+    # RPM-based distributions use 'site-packages' for Python packages
+    _copy_package_files(tempdir, paths, 'site-packages')
+
+
+def _deps_from_deb(tempdir, config, dinfo, pkg):
+    """Extract Python dependencies from Debian packages.
+
+    Args:
+        tempdir: Temporary directory to copy package files to
+        config: Build configuration
+        dinfo: DependencyInfo instance to track dependencies
+        pkg: Python package name (e.g., 'MarkupSafe', 'Jinja2', 'PyYAML')
+    """
+    # Convert Python package name to Debian package name
+    # Python packages are typically named python3-<lowercase-name>
+    # Handle special cases: PyYAML -> python3-yaml, MarkupSafe -> python3-markupsafe
+    pkg_lower = pkg.lower()
+    if pkg_lower == 'pyyaml':
+        deb_pkg_name = 'python3-yaml'
+    else:
+        deb_pkg_name = f'python3-{pkg_lower}'
+
+    # First, try to find the package using apt-cache
+    # This helps verify the package exists before trying to list its files
+    try:
+        res = subprocess.run(
+            ['apt-cache', 'show', deb_pkg_name],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError:
+        # Package not found, try alternative naming
+        log.warning(f"Package {deb_pkg_name} not found via apt-cache, trying dpkg -S")
+        # Try to search for files that might belong to this package
+        # Search for the Python module in site-packages
+        search_pattern = f'/usr/lib/python3*/dist-packages/{pkg.lower()}*'
+        try:
+            res = subprocess.run(
+                ['dpkg', '-S', search_pattern],
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            # dpkg -S output format: "package: /path/to/file"
+            deb_pkg_name = res.stdout.decode('utf8').split(':')[0].strip()
+        except subprocess.CalledProcessError as err:
+            log.error(f"Could not find Debian package for {pkg}")
+            log.error(f"Tried: {deb_pkg_name} and pattern search")
+            sys.exit(1)
+
+    # Get version information using dpkg-query
+    try:
+        res = subprocess.run(
+            ['dpkg-query', '-W', '-f=${Version}\\n', deb_pkg_name],
+            check=True,
+            stdout=subprocess.PIPE,
+        )
+        version = res.stdout.decode('utf8').strip()
+    except subprocess.CalledProcessError as err:
+        log.error(f"Could not query version for package {deb_pkg_name}: {err}")
+        sys.exit(1)
+
+    log.info(f"Debian Package: {deb_pkg_name} (version: {version})")
+    dinfo.add(
+        pkg,
+        deb_name=deb_pkg_name,
+        version=version,
+        package_source='deb',
+    )
+
+    # Get the list of files provided by the Debian package
+    try:
+        res = subprocess.run(
+            ['dpkg', '-L', deb_pkg_name],
+            check=True,
+            stdout=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError as err:
+        log.error(f"Could not list files for package {deb_pkg_name}: {err}")
+        sys.exit(1)
+
+    paths = [l.decode('utf8') for l in res.stdout.splitlines()]
+    # Debian-based distributions use 'dist-packages' for system-managed Python packages
+    # per Debian Python Policy: https://www.debian.org/doc/packaging-manuals/python-policy/
+    _copy_package_files(tempdir, paths, 'dist-packages')
 
 
 def generate_version_file(versioning_vars, dest):

src/cephadm/cephadm.py

@@ -1694,6 +1694,7 @@ def command_version(ctx):
     # type: (CephadmContext) -> int
     import importlib
     import zipimport
+    import zipfile
     import types
 
     vmod: Optional[types.ModuleType]
@@ -1750,10 +1751,17 @@ def command_version(ctx):
             out['bundled_packages'] = deps_info
         except OSError:
             pass
-        files = getattr(loader, '_files', {})
-        out['zip_root_entries'] = sorted(
-            {p.split('/')[0] for p in files.keys()}
-        )
+        # Use zipfile module to properly read the archive contents
+        # loader.archive contains the path to the zip file
+        try:
+            with zipfile.ZipFile(loader.archive, 'r') as zf:
+                files = zf.namelist()
+                out['zip_root_entries'] = sorted(
+                    {p.split('/')[0] for p in files if p}
+                )
+        except (OSError, zipfile.BadZipFile):
+            # Fallback to empty list if we can't read the zip
+            out['zip_root_entries'] = []
 
     json.dump(out, sys.stdout, indent=2)
     print()

src/cephadm/tests/build/test_cephadm_build.py

@@ -47,6 +47,16 @@
         'base_image': 'docker.io/library/ubuntu:24.04',
         'script': 'apt update && apt install -y python3-venv',
     },
+    'ubuntu-22.04-plusdeps': {
+        'name': 'cephadm-build-test:ubuntu-22-04-py3-deps',
+        'base_image': 'docker.io/library/ubuntu:22.04',
+        'script': 'apt update && apt install -y python3-jinja2 python3-yaml python3-markupsafe',
+    },
+    'ubuntu-24.04-plusdeps': {
+        'name': 'cephadm-build-test:ubuntu-24-04-py3-deps',
+        'base_image': 'docker.io/library/ubuntu:24.04',
+        'script': 'apt update && apt install -y python3-jinja2 python3-yaml python3-markupsafe',
+    },
 }
 
 BUILD_PY = 'src/cephadm/build.py'
@@ -193,7 +203,58 @@ def test_cephadm_build_from_rpms(env, source_dir, tmp_path):
     assert any(e.startswith('_cephadmmeta') for e in zre)
 
 
+@pytest.mark.parametrize(
+    'env',
+    [
+        'ubuntu-22.04-plusdeps',
+        'ubuntu-24.04-plusdeps',
+        'ubuntu-22.04',
+    ],
+)
+def test_cephadm_build_from_debs(env, source_dir, tmp_path):
+    res = build_in(
+        env,
+        source_dir,
+        tmp_path,
+        ['-Bdeb', '-SCEPH_GIT_VER=0', '-SCEPH_GIT_NICE_VER=foobar'],
+    )
+    if 'plusdeps' not in env:
+        assert res.returncode != 0
+        return
+    binary = tmp_path / 'cephadm'
+    assert binary.is_file()
+    res = subprocess.run(
+        [sys.executable, str(binary), 'version'],
+        stdout=subprocess.PIPE,
+    )
+    out = res.stdout.decode('utf8')
+    assert 'version' in out
+    assert 'foobar' in out
+    assert res.returncode == 0
+    res = subprocess.run(
+        [sys.executable, str(binary), 'version', '--verbose'],
+        stdout=subprocess.PIPE,
+    )
+    data = json.loads(res.stdout)
+    assert isinstance(data, dict)
+    assert 'bundled_packages' in data
+    assert all(v['package_source'] == 'deb' for v in data['bundled_packages'])
+    assert all(
+        v['name'] in ('Jinja2', 'MarkupSafe', 'PyYAML')
+        for v in data['bundled_packages']
+    )
+    assert all('requirements_entry' in v for v in data['bundled_packages'])
+    assert 'zip_root_entries' in data
+    zre = data['zip_root_entries']
+    assert any(_dist_info(e, 'Jinja2') for e in zre)
+    assert any(_dist_info(e, 'MarkupSafe') for e in zre)
+    assert any(e.startswith('jinja2') for e in zre)
+    assert any(e.startswith('markupsafe') for e in zre)
+    assert any(e.startswith('cephadmlib') for e in zre)
+    assert any(e.startswith('_cephadmmeta') for e in zre)
+
+
 def _dist_info(entry, name):
     return (
-        entry.startswith(entry) or entry.startswith(entry.lower())
+        entry.startswith(name) or entry.startswith(name.lower())
     ) and (entry.endswith('.dist-info') or entry.endswith('.egg-info'))