Merge pull request ceph#65176 from phlogistonjohn/jjm-cephadm-build-el10rpm

cephadm: fix building rpm-sourced cephadm zippapp on el10

Reviewed-by: Adam King <[email protected]>

Author: adk3798

src/cephadm/build.py

@@ -434,6 +434,47 @@ def _install_rpm_deps(tempdir, config):
     return dinfo
 
 
+def _gather_rpm_package_dirs(paths):
+    # = The easy way =
+    # the top_level.txt file can be used to determine where the python packages
+    # actually are. We need all of those and the meta-data dir (parent of
+    # top_level.txt) to be included in our zipapp
+    top_level = None
+    for path in paths:
+        if path.endswith('top_level.txt'):
+            top_level = pathlib.Path(path)
+    if top_level:
+        meta_dir = top_level.parent
+        pkg_dirs = [
+            top_level.parent.parent / p
+            for p in top_level.read_text().splitlines()
+        ]
+        return meta_dir, pkg_dirs
+    # = The hard way =
+    # loop through the directories to find the .dist-info dir (containing the
+    # mandatory METADATA file, according to the spec) and once we know the
+    # location of dist info we find the sibling paths from the rpm listing
+    dist_info = None
+    ppaths = []
+    for path in paths:
+        ppath = pathlib.Path(path)
+        ppaths.append(ppath)
+        if ppath.name == 'METADATA' and ppath.parent.name.endswith('.dist-info'):
+            dist_info = ppath.parent
+            break
+    if not dist_info:
+        raise ValueError('no .dist-info METADATA found')
+    if not dist_info.parent.name == 'site-packages':
+        raise ValueError(
+            'unexpected parent directory (not site-packages):'
+            f' {dist_info.parent.name}'
+        )
+    siblings = [
+        p for p in ppaths if p.parent == dist_info.parent and p != dist_info
+    ]
+    return dist_info, siblings
+
+
 def _deps_from_rpm(tempdir, config, dinfo, pkg):
     # first, figure out what rpm provides a particular python lib
     dist = f'python3.{sys.version_info.minor}dist({pkg})'.lower()
@@ -469,20 +510,7 @@ def _deps_from_rpm(tempdir, config, dinfo, pkg):
         ['rpm', '-ql', rpmname], check=True, stdout=subprocess.PIPE
     )
     paths = [l.decode('utf8') for l in res.stdout.splitlines()]
-    # the top_level.txt file can be used to determine where the python packages
-    # actually are. We need all of those and the meta-data dir (parent of
-    # top_level.txt) to be included in our zipapp
-    top_level = None
-    for path in paths:
-        if path.endswith('top_level.txt'):
-            top_level = pathlib.Path(path)
-    if not top_level:
-        raise ValueError('top_level not found')
-    meta_dir = top_level.parent
-    pkg_dirs = [
-        top_level.parent.parent / p
-        for p in top_level.read_text().splitlines()
-    ]
+    meta_dir, pkg_dirs = _gather_rpm_package_dirs(paths)
     meta_dest = tempdir / meta_dir.name
     log.info(f"Copying {meta_dir} to {meta_dest}")
     # copy the meta data directory

src/cephadm/tests/build/test_cephadm_build.py

@@ -12,34 +12,39 @@
 
 
 CONTAINERS = {
-    'centos-8': {
-        'name': 'cephadm-build-test:centos8-py36',
-        'base_image': 'quay.io/centos/centos:stream8',
-        'script': 'dnf install -y python36',
-    },
     'centos-9': {
         'name': 'cephadm-build-test:centos9-py3',
         'base_image': 'quay.io/centos/centos:stream9',
         'script': 'dnf install -y python3',
     },
-    'centos-8-plusdeps': {
-        'name': 'cephadm-build-test:centos8-py36-deps',
-        'base_image': 'quay.io/centos/centos:stream8',
-        'script': 'dnf install -y python36 python3-jinja2 python3-pyyaml',
+    'centos-10': {
+        'name': 'cephadm-build-test:centos10-py3',
+        'base_image': 'quay.io/centos/centos:stream10',
+        'script': 'dnf install -y python3',
     },
     'centos-9-plusdeps': {
         'name': 'cephadm-build-test:centos9-py3-deps',
         'base_image': 'quay.io/centos/centos:stream9',
         'script': 'dnf install -y python3 python3-jinja2 python3-pyyaml',
     },
+    'centos-10-plusdeps': {
+        'name': 'cephadm-build-test:centos10-py3-deps',
+        'base_image': 'quay.io/centos/centos:stream10',
+        'script': 'dnf install -y python3 python3-jinja2 python3-pyyaml',
+    },
     'ubuntu-20.04': {
         'name': 'cephadm-build-test:ubuntu-20-04-py3',
-        'base_image': 'quay.io/library/ubuntu:20.04',
+        'base_image': 'docker.io/library/ubuntu:20.04',
         'script': 'apt update && apt install -y python3-venv',
     },
     'ubuntu-22.04': {
         'name': 'cephadm-build-test:ubuntu-22-04-py3',
-        'base_image': 'quay.io/library/ubuntu:22.04',
+        'base_image': 'docker.io/library/ubuntu:22.04',
+        'script': 'apt update && apt install -y python3-venv',
+    },
+    'ubuntu-24.04': {
+        'name': 'cephadm-build-test:ubuntu-24-04-py3',
+        'base_image': 'docker.io/library/ubuntu:24.04',
         'script': 'apt update && apt install -y python3-venv',
     },
 }
@@ -95,10 +100,11 @@ def source_dir():
 @pytest.mark.parametrize(
     'env',
     [
-        'centos-8',
         'centos-9',
+        'centos-10',
         'ubuntu-20.04',
         'ubuntu-22.04',
+        'ubuntu-24.04',
     ],
 )
 def test_cephadm_build(env, source_dir, tmp_path):
@@ -128,8 +134,8 @@ def test_cephadm_build(env, source_dir, tmp_path):
     assert all('requirements_entry' in v for v in data['bundled_packages'])
     assert 'zip_root_entries' in data
     zre = data['zip_root_entries']
-    assert any(e.startswith('Jinja2') for e in zre)
-    assert any(e.startswith('MarkupSafe') for e in zre)
+    assert any(_dist_info(e, 'Jinja2') for e in zre)
+    assert any(_dist_info(e, 'MarkupSafe') for e in zre)
     assert any(e.startswith('jinja2') for e in zre)
     assert any(e.startswith('markupsafe') for e in zre)
     assert any(e.startswith('cephadmlib') for e in zre)
@@ -139,8 +145,8 @@ def test_cephadm_build(env, source_dir, tmp_path):
 @pytest.mark.parametrize(
     'env',
     [
-        'centos-8-plusdeps',
         'centos-9-plusdeps',
+        'centos-10-plusdeps',
         'centos-9',
     ],
 )
@@ -155,11 +161,6 @@ def test_cephadm_build_from_rpms(env, source_dir, tmp_path):
         assert res.returncode != 0
         return
     binary = tmp_path / 'cephadm'
-    if 'centos-8' in env and sys.version_info[:2] >= (3, 10):
-        # The version of markupsafe in centos 8 is incompatible with
-        # python>=3.10 due to changes in the stdlib therefore we can't execute
-        # the cephadm binary, so we quit the test early.
-        return
     assert binary.is_file()
     res = subprocess.run(
         [sys.executable, str(binary), 'version'],
@@ -184,9 +185,15 @@ def test_cephadm_build_from_rpms(env, source_dir, tmp_path):
     assert all('requirements_entry' in v for v in data['bundled_packages'])
     assert 'zip_root_entries' in data
     zre = data['zip_root_entries']
-    assert any(e.startswith('Jinja2') for e in zre)
-    assert any(e.startswith('MarkupSafe') for e in zre)
+    assert any(_dist_info(e, 'Jinja2') for e in zre)
+    assert any(_dist_info(e, 'MarkupSafe') for e in zre)
     assert any(e.startswith('jinja2') for e in zre)
     assert any(e.startswith('markupsafe') for e in zre)
     assert any(e.startswith('cephadmlib') for e in zre)
     assert any(e.startswith('_cephadmmeta') for e in zre)
+
+
+def _dist_info(entry, name):
+    return (
+        entry.startswith(entry) or entry.startswith(entry.lower())
+    ) and (entry.endswith('.dist-info') or entry.endswith('.egg-info'))