diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 984231dfb1..f2392335af 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -25,6 +25,7 @@ from teuthology.suite.merge import config_merge from teuthology.suite.build_matrix import build_matrix from teuthology.suite.placeholder import substitute_placeholders, dict_templ +from teuthology.util.containers import container_image_for_hash, container_image_exists from teuthology.util.time import parse_offset, parse_timestamp, TIMESTAMP_FMT log = logging.getLogger(__name__) @@ -549,6 +550,22 @@ def collect_jobs(self, arch, configs, newest=False, limit=0): # no point in continuing the search if newest: return jobs_missing_packages, [] + for task_dict in parsed_yaml.get('tasks', []): + for key in task_dict.keys(): + if 'cephadm' in key: + # Check for our default image + if not container_image_for_hash(sha1): + jobs_missing_packages.append(job) + if newest: + return jobs_missing_packages, [] + if key == 'cephadm': + if image := (task_dict.get(key) or {}).get('image'): + # Check each image that the job would use + if not container_image_exists(image): + log.error(f"Required container image missing: {image}") + jobs_missing_packages.append(job) + if newest: + return jobs_missing_packages, [] jobs_to_schedule.append(job) return jobs_missing_packages, jobs_to_schedule diff --git a/teuthology/suite/util.py b/teuthology/suite/util.py index cc884ebf90..251dcc7f75 100644 --- a/teuthology/suite/util.py +++ b/teuthology/suite/util.py @@ -25,9 +25,6 @@ log = logging.getLogger(__name__) -CONTAINER_DISTRO = 'centos/9' # the one to check for build_complete -CONTAINER_FLAVOR = 'default' - def fetch_repos(branch, test_name, dry_run, commit=None): """ @@ -245,17 +242,11 @@ def package_version_for_hash(hash, flavor='default', distro='rhel', ), ) - if (bp.distro == CONTAINER_DISTRO and bp.flavor == CONTAINER_FLAVOR and - not bp.build_complete): - log.info("Container build incomplete") - return None - try: return bp.version except VersionNotFoundError: return None - def get_arch(machine_type): """ Based on a given machine_type, return its architecture by querying the lock diff --git a/teuthology/util/containers.py b/teuthology/util/containers.py new file mode 100644 index 0000000000..661422049a --- /dev/null +++ b/teuthology/util/containers.py @@ -0,0 +1,76 @@ +import functools +import logging +import re +import requests + +log = logging.getLogger(__name__) + +# Our container images use a certain base image and flavor by default; those +# values are reflected below. If different values are used, they are appended +# to the image name. +DEFAULT_CONTAINER_BASE = 'centos:9' +DEFAULT_CONTAINER_FLAVOR = 'default' +DEFAULT_CONTAINER_IMAGE='quay.ceph.io/ceph-ci/ceph:{sha1}' +CONTAINER_REGEXP = re.compile( + r"((?P[a-zA-Z0-9._-]+)/)?((?P[a-zA-Z0-9_-]+)/)?((?P[a-zA-Z0-9_-]+))?(:(?P[a-zA-Z0-9._-]+))?" +) + + +def resolve_container_image(image: str): + """ + Given an image locator that is potentially incomplete, construct a qualified version. + + ':tag' -> 'quay.ceph.io/ceph-ci/ceph:tag' + 'image:tag' -> 'quay.ceph.io/ceph-ci/image:tag' + 'org/image:tag' -> 'quay.ceph.io/org/image:tag' + 'example.com/org/image:tag' -> 'example.com/org/image:tag' + """ + try: + (image_long, tag) = image.split(':') + except ValueError: + raise ValueError(f"Container image spec missing tag: {image}") from None + domain = 'quay.ceph.io' + org = 'ceph-ci' + image = 'ceph' + image_split = image_long.split('/') + assert len(image_split) <= 3 + match len(image_split): + case 3: + (domain, org, image) = image_split + case 2: + (org, image) = image_split + case _: + if image_split[0]: + image = image_split[0] + return f"{domain}/{org}/{image}:{tag}" + + +@functools.lru_cache() +def container_image_exists(image: str): + """ + Use the Quay API to check for the existence of a container image. + Only tested with Quay registries. + """ + match = re.match(CONTAINER_REGEXP, image) + assert match + obj = match.groupdict() + url = f"https://{obj['domain']}/api/v1/repository/{obj['org']}/{obj['image']}/tag?filter_tag_name=eq:{obj['tag']}" + log.info(f"Checking for container existence at: {url}") + resp = requests.get(url) + return resp.ok and len(resp.json().get('tags')) >= 1 + + +def container_image_for_hash(hash: str, flavor='default', base_image='centos:9'): + """ + Given a sha1 and optionally a base image and flavor, attempt to return a container image locator. + """ + tag = hash + if base_image != DEFAULT_CONTAINER_BASE: + tag = f"{tag}-{base_image.replace(':', '-')}" + if flavor != DEFAULT_CONTAINER_FLAVOR: + tag = f"{tag}-{flavor}" + image_spec = resolve_container_image(f":{tag}") + if container_image_exists(image_spec): + return image_spec + else: + log.error(f"Container image not found for hash '{hash}'") diff --git a/teuthology/util/test/test_containers.py b/teuthology/util/test/test_containers.py new file mode 100644 index 0000000000..8c270f7840 --- /dev/null +++ b/teuthology/util/test/test_containers.py @@ -0,0 +1,50 @@ +import pytest + +from unittest.mock import patch + +from teuthology.util import containers + +@pytest.mark.parametrize( + 'input, expected', + [ + (':hash', 'quay.ceph.io/ceph-ci/ceph:hash'), + ('image:hash', 'quay.ceph.io/ceph-ci/image:hash'), + ('org/image:hash', 'quay.ceph.io/org/image:hash'), + ('example.com/org/image:hash', 'example.com/org/image:hash'), + ('image', ValueError), + ('org/image', ValueError), + ('domain.net/org/image', ValueError), + ] +) +def test_resolve_container_image(input, expected): + if isinstance(expected, str): + assert expected == containers.resolve_container_image(input) + else: + with pytest.raises(expected): + containers.resolve_container_image(input) + +@pytest.mark.parametrize( + 'image, url', + [ + ('example.com/org/image:tag', 'https://example.com/api/v1/repository/org/image/tag?filter_tag_name=eq:tag'), + ] +) +def test_container_image_exists(image, url): + with patch("teuthology.util.containers.requests.get") as m_get: + containers.container_image_exists(image) + m_get.assert_called_once_with(url) + + +@pytest.mark.parametrize( + 'hash, flavor, base_image, rci_input', + [ + ('hash', 'flavor', 'base-image', ':hash-base-image-flavor'), + ('hash', 'default', 'centos:9', ':hash'), + ('hash', 'default', 'rockylinux-10', ':hash-rockylinux-10'), + ] +) +def test_container_image_for_hash(hash, flavor, base_image, rci_input): + with patch('teuthology.util.containers.resolve_container_image') as m_rci: + with patch('teuthology.util.containers.container_image_exists'): + containers.container_image_for_hash(hash, flavor, base_image) + m_rci.assert_called_once_with(rci_input)