suite: Check for existence of container images

teuthology/suite/run.py

@@ -25,6 +25,7 @@
 from teuthology.suite.merge import config_merge
 from teuthology.suite.build_matrix import build_matrix
 from teuthology.suite.placeholder import substitute_placeholders, dict_templ
+from teuthology.util.containers import container_image_for_hash, container_image_exists
 from teuthology.util.time import parse_offset, parse_timestamp, TIMESTAMP_FMT
 
 log = logging.getLogger(__name__)
@@ -549,6 +550,22 @@ def collect_jobs(self, arch, configs, newest=False, limit=0):
                     # no point in continuing the search
                     if newest:
                         return jobs_missing_packages, []
+                for task_dict in parsed_yaml.get('tasks', []):
+                    for key in task_dict.keys():
+                        if 'cephadm' in key:
+                            # Check for our default image
+                            if not container_image_for_hash(sha1):
+                                jobs_missing_packages.append(job)
+                                if newest:
+                                    return jobs_missing_packages, []
+                        if key == 'cephadm':
+                            if image := (task_dict.get(key) or {}).get('image'):
+                                # Check each image that the job would use
+                                if not container_image_exists(image):
+                                    log.error(f"Required container image missing: {image}")
+                                    jobs_missing_packages.append(job)
+                                    if newest:
+                                        return jobs_missing_packages, []
 
             jobs_to_schedule.append(job)
         return jobs_missing_packages, jobs_to_schedule

teuthology/suite/util.py

@@ -25,9 +25,6 @@
 
 log = logging.getLogger(__name__)
 
-CONTAINER_DISTRO = 'centos/9'       # the one to check for build_complete
-CONTAINER_FLAVOR = 'default'
-
 
 def fetch_repos(branch, test_name, dry_run, commit=None):
     """
@@ -245,17 +242,11 @@ def package_version_for_hash(hash, flavor='default', distro='rhel',
         ),
     )
 
-    if (bp.distro == CONTAINER_DISTRO and bp.flavor == CONTAINER_FLAVOR and 
-            not bp.build_complete):
-        log.info("Container build incomplete")
-        return None
-
     try:
         return bp.version
     except VersionNotFoundError:
         return None
 
-
 def get_arch(machine_type):
     """
     Based on a given machine_type, return its architecture by querying the lock

teuthology/util/containers.py

@@ -0,0 +1,76 @@
+import functools
+import logging
+import re
+import requests
+
+log = logging.getLogger(__name__)
+
+# Our container images use a certain base image and flavor by default; those
+# values are reflected below. If different values are used, they are appended
+# to the image name.
+DEFAULT_CONTAINER_BASE = 'centos:9'
+DEFAULT_CONTAINER_FLAVOR = 'default'
+DEFAULT_CONTAINER_IMAGE='quay.ceph.io/ceph-ci/ceph:{sha1}'
+CONTAINER_REGEXP = re.compile(
+    r"((?P<domain>[a-zA-Z0-9._-]+)/)?((?P<org>[a-zA-Z0-9_-]+)/)?((?P<image>[a-zA-Z0-9_-]+))?(:(?P<tag>[a-zA-Z0-9._-]+))?"
+)
+
+
+def resolve_container_image(image: str):
+    """
+    Given an image locator that is potentially incomplete, construct a qualified version.
+
+    ':tag' -> 'quay.ceph.io/ceph-ci/ceph:tag'
+    'image:tag' -> 'quay.ceph.io/ceph-ci/image:tag'
+    'org/image:tag' -> 'quay.ceph.io/org/image:tag'
+    'example.com/org/image:tag' -> 'example.com/org/image:tag'
+    """
+    try:
+        (image_long, tag) = image.split(':')
+    except ValueError:
+        raise ValueError(f"Container image spec missing tag: {image}") from None
+    domain = 'quay.ceph.io'
+    org = 'ceph-ci'
+    image = 'ceph'
+    image_split = image_long.split('/')
+    assert len(image_split) <= 3
+    match len(image_split):
+        case 3:
+            (domain, org, image) = image_split
+        case 2:
+            (org, image) = image_split
+        case _:
+            if image_split[0]:
+                image = image_split[0]
+    return f"{domain}/{org}/{image}:{tag}"
+
+
+@functools.lru_cache()
+def container_image_exists(image: str):
+    """
+    Use the Quay API to check for the existence of a container image.
+    Only tested with Quay registries.
+    """
+    match = re.match(CONTAINER_REGEXP, image)
+    assert match
+    obj = match.groupdict()
+    url = f"https://{obj['domain']}/api/v1/repository/{obj['org']}/{obj['image']}/tag?filter_tag_name=eq:{obj['tag']}"
+    log.info(f"Checking for container existence at: {url}")
+    resp = requests.get(url)
+    return resp.ok and len(resp.json().get('tags')) >= 1
+
+
+def container_image_for_hash(hash: str, flavor='default', base_image='centos:9'):
+    """
+    Given a sha1 and optionally a base image and flavor, attempt to return a container image locator.
+    """
+    tag = hash
+    if base_image != DEFAULT_CONTAINER_BASE:
+        tag = f"{tag}-{base_image.replace(':', '-')}"
+    if flavor != DEFAULT_CONTAINER_FLAVOR:
+        tag = f"{tag}-{flavor}"
+    image_spec = resolve_container_image(f":{tag}")
+    if container_image_exists(image_spec):
+        return image_spec
+    else:
+        log.error(f"Container image not found for hash '{hash}'")

teuthology/util/test/test_containers.py

@@ -0,0 +1,50 @@
+import pytest
+
+from unittest.mock import patch
+
+from teuthology.util import containers
+
+@pytest.mark.parametrize(
+    'input, expected',
+    [
+        (':hash', 'quay.ceph.io/ceph-ci/ceph:hash'),
+        ('image:hash', 'quay.ceph.io/ceph-ci/image:hash'),
+        ('org/image:hash', 'quay.ceph.io/org/image:hash'),
+        ('example.com/org/image:hash', 'example.com/org/image:hash'),
+        ('image', ValueError),
+        ('org/image', ValueError),
+        ('domain.net/org/image', ValueError),
+    ]
+)
+def test_resolve_container_image(input, expected):
+    if isinstance(expected, str):
+        assert expected == containers.resolve_container_image(input)
+    else:
+        with pytest.raises(expected):
+            containers.resolve_container_image(input)
+
+@pytest.mark.parametrize(
+    'image, url',
+    [
+        ('example.com/org/image:tag', 'https://example.com/api/v1/repository/org/image/tag?filter_tag_name=eq:tag'),
+    ]
+)
+def test_container_image_exists(image, url):
+    with patch("teuthology.util.containers.requests.get") as m_get:
+        containers.container_image_exists(image)
+        m_get.assert_called_once_with(url)
+
+
+@pytest.mark.parametrize(
+    'hash, flavor, base_image, rci_input',
+    [
+        ('hash', 'flavor', 'base-image', ':hash-base-image-flavor'),
+        ('hash', 'default', 'centos:9', ':hash'),
+        ('hash', 'default', 'rockylinux-10', ':hash-rockylinux-10'),
+    ]
+)
+def test_container_image_for_hash(hash, flavor, base_image, rci_input):
+    with patch('teuthology.util.containers.resolve_container_image') as m_rci:
+        with patch('teuthology.util.containers.container_image_exists'):
+            containers.container_image_for_hash(hash, flavor, base_image)
+            m_rci.assert_called_once_with(rci_input)