From 9a641cbe5fdaa0485ec1dea8f1789cff685ea9b4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 20 Aug 2025 20:31:40 +0200 Subject: [PATCH 1/4] upgrade to registry v3 --- Makefile | 2 +- packages/pytest-simcore/src/pytest_simcore/docker_registry.py | 2 +- services/docker-compose-ops-registry.yml | 2 +- tests/e2e/docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7c8cfbf29acd..61d42b4ff96e 100644 --- a/Makefile +++ b/Makefile @@ -707,7 +707,7 @@ local-registry: .env ## creates a local docker registry and configure simcore to --publish 5000:5000 \ --volume $(LOCAL_REGISTRY_VOLUME):/var/lib/registry \ --name $(LOCAL_REGISTRY_HOSTNAME) \ - registry:2) + registry:3) # WARNING: environment file .env is now setup to use local registry on port 5000 without any security (take care!)... @echo REGISTRY_AUTH=False >> .env diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index 0f1a5271685d..20ef5dc593c7 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -43,7 +43,7 @@ def docker_registry(keep_docker_up: bool) -> Iterator[str]: print("Warning: docker registry is already up!") except Exception: # pylint: disable=broad-except container = docker_client.containers.run( - "registry:2", + "registry:3", ports={"5000": "5000"}, name="pytest_registry", environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"], diff --git a/services/docker-compose-ops-registry.yml b/services/docker-compose-ops-registry.yml index ffc4b5d206c2..4b0c087c6325 100644 --- a/services/docker-compose-ops-registry.yml +++ b/services/docker-compose-ops-registry.yml @@ -4,7 +4,7 @@ version: "3.7" services: registry: - image: registry:2 + image: registry:3 container_name: registry init: true environment: diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index 245eb07482b6..e319d6275966 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -1,6 +1,6 @@ services: registry: - image: registry:2 + image: registry:3 restart: always ports: - "5000:5000" From aad5d6dbea45be028b28381eb8fd765674578506 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 20 Aug 2025 21:18:06 +0200 Subject: [PATCH 2/4] delete unused stuff --- scripts/metrics/Makefile | 11 -- .../compute_list_of_images_in_registry.py | 144 ------------------ scripts/metrics/requirements.txt | 5 - 3 files changed, 160 deletions(-) delete mode 100644 scripts/metrics/Makefile delete mode 100755 scripts/metrics/compute_list_of_images_in_registry.py delete mode 100644 scripts/metrics/requirements.txt diff --git a/scripts/metrics/Makefile b/scripts/metrics/Makefile deleted file mode 100644 index 3453cfe3cfb6..000000000000 --- a/scripts/metrics/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -.DEFAULT_GOAL := install - -SHELL := /bin/bash - -install: - # creating python virtual environment - @python3 -m venv .venv - # installing python dependencies - @.venv/bin/pip install --upgrade pip setuptools wheel - @.venv/bin/pip install -r requirements.txt - # activate the python virtual environment by running: ```source .venv/bin/activate``` diff --git a/scripts/metrics/compute_list_of_images_in_registry.py b/scripts/metrics/compute_list_of_images_in_registry.py deleted file mode 100755 index 518c7932b5f2..000000000000 --- a/scripts/metrics/compute_list_of_images_in_registry.py +++ /dev/null @@ -1,144 +0,0 @@ -#! /usr/bin/env python3 - -import asyncio -import json -from collections import defaultdict, deque -from datetime import date, datetime -from pathlib import Path -from pprint import pformat - -import typer -from httpx import URL, AsyncClient - -N = len("2020-10-09T12:28:14.7710") - - -async def get_repos(client): - r = await client.get( - "/_catalog", - ) - r.raise_for_status() - list_of_repositories = r.json()["repositories"] - typer.secho( - f"got the list of {len(list_of_repositories)} repositories from the registry" - ) - filtered_list_of_repositories = list( - filter( - lambda repo: repo.startswith("simcore/services/dynamic/") - or repo.startswith("simcore/services/comp/"), - list_of_repositories, - ) - ) - return filtered_list_of_repositories - - -async def list_images_in_registry( - endpoint: URL, - username: str, - password: str, - from_date: datetime | None, - to_date: datetime, -) -> dict[str, list[tuple[str, str, str, str]]]: - if not from_date: - from_date = datetime(year=2000, month=1, day=1) - typer.secho( - f"listing images from {from_date} to {to_date} from {endpoint}", - fg=typer.colors.YELLOW, - ) - - list_of_images_in_date_range = defaultdict(list) - - async with AsyncClient( - base_url=endpoint.join("v2"), auth=(username, password), http2=True - ) as client: - list_of_repositories = await get_repos(client) - - with typer.progressbar( - list_of_repositories, label="Processing repositories" - ) as progress: - for repo in progress: - r = await client.get(f"/{repo}/tags/list") - r.raise_for_status() - list_of_tags = [tag for tag in r.json()["tags"] if tag != "latest"] - - # we go in reverse order, so the first that does not go in the date range will stop the loop - for tag in reversed(list_of_tags): - r = await client.get(f"/{repo}/manifests/{tag}") - r.raise_for_status() - manifest = r.json() - # manifest[history] contains all the blobs, taking the latest one corresponds to the image creation date - history = manifest["history"] - tag_creation_dates = deque() - for blob in history: - v1_comp = json.loads(blob["v1Compatibility"]) - tag_creation_dates.append( - datetime.strptime( - v1_comp["created"][:N], "%Y-%m-%dT%H:%M:%S.%f" - ) - ) - tag_last_date = sorted(tag_creation_dates)[-1] - # check this service is in the time range - if tag_last_date < from_date or tag_last_date > to_date: - break - - # get the image labels from the last blob (same as director does) - v1_comp = json.loads(history[0]["v1Compatibility"]) - container_config = v1_comp.get( - "container_config", v1_comp["config"] - ) - - simcore_labels = {} - for label_key, label_value in container_config["Labels"].items(): - if label_key.startswith("io.simcore"): - simcore_labels.update(json.loads(label_value)) - - list_of_images_in_date_range[repo].append( - ( - tag, - simcore_labels["name"], - simcore_labels["description"], - simcore_labels["type"], - ) - ) - typer.secho( - f"Completed. Found {len(list_of_images_in_date_range)} created between {from_date} and {to_date}", - fg=typer.colors.YELLOW, - ) - typer.secho(f"{pformat(list_of_images_in_date_range)}") - - return list_of_images_in_date_range - - -def main( - endpoint: str, - username: str, - password: str = typer.Option(..., prompt=True, hide_input=True), - from_date: datetime | None = typer.Option(None, formats=["%Y-%m-%d"]), - to_date: datetime = typer.Option(f"{date.today()}", formats=["%Y-%m-%d"]), - markdown: bool = typer.Option(False), -): - endpoint_url = URL(endpoint) - list_of_images: dict[str, list[tuple[str, str, str, str]]] = asyncio.run( - list_images_in_registry(endpoint_url, username, password, from_date, to_date) - ) - - if markdown: - output_file = Path.cwd() / f"{endpoint_url.host}.md" - with output_file.open("w") as fp: - fp.write( - f"# {endpoint_url.host}: Services added between {from_date} and {to_date}\n\n" - ) - fp.write("| Service | Version(s) | Name | Description | Type |\n") - fp.write("| ------- | ---------- | ---- | ----------- | ---- |\n") - for repo, repo_details in list_of_images.items(): - for index, (version, name, description, service_type) in enumerate( - repo_details - ): - filtered_description = description.strip().replace("\n", "") - fp.write( - f"| {repo if index == 0 else ''} | {version} | {name if index == 0 else ''} | {filtered_description if index == 0 else ''} | {('Dynamic service' if service_type == 'dynamic' else 'Computational service') if index == 0 else ''} |\n" - ) - - -if __name__ == "__main__": - typer.run(main) diff --git a/scripts/metrics/requirements.txt b/scripts/metrics/requirements.txt deleted file mode 100644 index fd90784f7b6d..000000000000 --- a/scripts/metrics/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -black -httpx[http2] -pydantic[email] -pylint -typer From ea2576321b94dc251e7d56ddff690762afcb382c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 20 Aug 2025 21:18:28 +0200 Subject: [PATCH 3/4] be compatible with docker image manifest v2 schema 2 --- .../simcore_service_director/core/errors.py | 6 ++ .../registry_proxy.py | 62 ++++++++++++++----- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/services/director/src/simcore_service_director/core/errors.py b/services/director/src/simcore_service_director/core/errors.py index c7113baa4021..b983d520734d 100644 --- a/services/director/src/simcore_service_director/core/errors.py +++ b/services/director/src/simcore_service_director/core/errors.py @@ -17,6 +17,12 @@ class ServiceNotAvailableError(DirectorRuntimeError): msg_template: str = "Service {service_name}:{service_tag} is not available" +class DockerRegistryUnsupportedManifestSchemaVersionError(DirectorRuntimeError): + msg_template: str = ( + "Docker registry schema version {version} issue with {service_name}:{service_tag}" + ) + + class ServiceUUIDNotFoundError(DirectorRuntimeError): msg_template: str = "Service with uuid {service_uuid} was not found" diff --git a/services/director/src/simcore_service_director/registry_proxy.py b/services/director/src/simcore_service_director/registry_proxy.py index 177460f915c0..bd80228d2fc7 100644 --- a/services/director/src/simcore_service_director/registry_proxy.py +++ b/services/director/src/simcore_service_director/registry_proxy.py @@ -25,6 +25,7 @@ from .constants import DIRECTOR_SIMCORE_SERVICES_PREFIX from .core.errors import ( DirectorRuntimeError, + DockerRegistryUnsupportedManifestSchemaVersionError, RegistryConnectionError, ServiceNotAvailableError, ) @@ -375,22 +376,55 @@ async def get_image_labels( app: FastAPI, image: str, tag: str, *, update_cache=False ) -> tuple[dict[str, str], str | None]: """Returns image labels and the image manifest digest""" + with log_context(_logger, logging.DEBUG, msg=f"get {image}:{tag} labels"): + request_result, headers = await registry_request( + app, + path=f"{image}/manifests/{tag}", + method="GET", + use_cache=not update_cache, + ) - _logger.debug("getting image labels of %s:%s", image, tag) - path = f"{image}/manifests/{tag}" - request_result, headers = await registry_request( - app, path=path, method="GET", use_cache=not update_cache - ) - v1_compatibility_key = json_loads(request_result["history"][0]["v1Compatibility"]) - container_config: dict[str, Any] = v1_compatibility_key.get( - "container_config", v1_compatibility_key["config"] - ) - labels: dict[str, str] = container_config["Labels"] - - headers = headers or {} - manifest_digest: str | None = headers.get(_DOCKER_CONTENT_DIGEST_HEADER, None) + schema_version = request_result["schemaVersion"] + labels: dict[str, str] = {} + match schema_version: + case 2: + # Image Manifest Version 2, Schema 2 -> defaults in registries v3 (https://distribution.github.io/distribution/spec/manifest-v2-2/) + media_type = request_result["mediaType"] + if ( + media_type + == "application/vnd.docker.distribution.manifest.list.v2+json" + ): + raise DockerRegistryUnsupportedManifestSchemaVersionError( + version=schema_version, + service_name=image, + service_tag=tag, + reason="Multiple architectures images are currently not supported and need to be implemented", + ) + config_digest = request_result["config"]["digest"] + # Fetch the config blob + config_result, _ = await registry_request( + app, + path=f"{image}/blobs/{config_digest}", + method="GET", + use_cache=not update_cache, + ) + labels = config_result.get("config", {}).get("Labels", {}) + case 1: + # Image Manifest Version 2, Schema 1 deprecated in docker hub since 2024-11-04 + v1_compatibility_key = json_loads( + request_result["history"][0]["v1Compatibility"] + ) + container_config: dict[str, Any] = v1_compatibility_key.get( + "container_config", v1_compatibility_key.get("config", {}) + ) + labels = container_config.get("Labels", {}) + case _: + raise DockerRegistryUnsupportedManifestSchemaVersionError( + version=schema_version, service_name=image, service_tag=tag + ) - _logger.debug("retrieved labels of image %s:%s", image, tag) + headers = headers or {} + manifest_digest: str | None = headers.get(_DOCKER_CONTENT_DIGEST_HEADER, None) return (labels, manifest_digest) From 59ad06bd61f905ccb1cb9d45ce19d6fa9f399f6d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:10:57 +0200 Subject: [PATCH 4/4] upgraded director-v0 to be compatible with both registries --- .../src/pytest_simcore/docker_registry.py | 38 ++++++++++++------- .../tests/unit/fixtures/fake_services.py | 17 +++++---- .../tests/unit/test_registry_proxy.py | 28 ++++++++++++-- 3 files changed, 59 insertions(+), 24 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py index 20ef5dc593c7..f54dea4cadb1 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_registry.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_registry.py @@ -22,35 +22,45 @@ from .helpers.host import get_localhost_ip -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) @pytest.fixture(scope="session") def docker_registry(keep_docker_up: bool) -> Iterator[str]: + """sets up and runs a docker registry container locally and returns its URL""" + yield from _docker_registry_impl(keep_docker_up, registry_version="3") + + +@pytest.fixture(scope="session") +def docker_registry_v2() -> Iterator[str]: + """sets up and runs a docker registry v2 container locally and returns its URL""" + yield from _docker_registry_impl(keep_docker_up=False, registry_version="2") + + +def _docker_registry_impl(keep_docker_up: bool, registry_version: str) -> Iterator[str]: """sets up and runs a docker registry container locally and returns its URL""" # run the registry outside of the stack docker_client = docker.from_env() # try to login to private registry host = "127.0.0.1" - port = 5000 + port = 5000 if registry_version == "3" else 5001 url = f"{host}:{port}" + container_name = f"pytest_registry_v{registry_version}" + volume_name = f"pytest_registry_v{registry_version}_data" + container = None try: docker_client.login(registry=url, username="simcore") - container = docker_client.containers.list(filters={"name": "pytest_registry"})[ - 0 - ] + container = docker_client.containers.list(filters={"name": container_name})[0] print("Warning: docker registry is already up!") except Exception: # pylint: disable=broad-except container = docker_client.containers.run( - "registry:3", - ports={"5000": "5000"}, - name="pytest_registry", + f"registry:{registry_version}", + ports={"5000": port}, + name=container_name, environment=["REGISTRY_STORAGE_DELETE_ENABLED=true"], restart_policy={"Name": "always"}, - volumes={ - "pytest_registry_data": {"bind": "/var/lib/registry", "mode": "rw"} - }, + volumes={volume_name: {"bind": "/var/lib/registry", "mode": "rw"}}, detach=True, ) @@ -79,9 +89,9 @@ def docker_registry(keep_docker_up: bool) -> Iterator[str]: os.environ["REGISTRY_SSL"] = "False" os.environ["REGISTRY_AUTH"] = "False" # the registry URL is how to access from the container (e.g. for accessing the API) - os.environ["REGISTRY_URL"] = f"{get_localhost_ip()}:5000" + os.environ["REGISTRY_URL"] = f"{get_localhost_ip()}:{port}" # the registry PATH is how the docker engine shall access the images (usually same as REGISTRY_URL but for testing) - os.environ["REGISTRY_PATH"] = "127.0.0.1:5000" + os.environ["REGISTRY_PATH"] = f"127.0.0.1:{port}" os.environ["REGISTRY_USER"] = "simcore" os.environ["REGISTRY_PW"] = "" @@ -124,7 +134,7 @@ def registry_settings( @tenacity.retry( wait=tenacity.wait_fixed(2), stop=tenacity.stop_after_delay(20), - before_sleep=tenacity.before_sleep_log(log, logging.INFO), + before_sleep=tenacity.before_sleep_log(_logger, logging.INFO), reraise=True, ) def wait_till_registry_is_responsive(url: str) -> bool: diff --git a/services/director/tests/unit/fixtures/fake_services.py b/services/director/tests/unit/fixtures/fake_services.py index 1edb799ee9cb..af867d08d920 100644 --- a/services/director/tests/unit/fixtures/fake_services.py +++ b/services/director/tests/unit/fixtures/fake_services.py @@ -117,7 +117,6 @@ async def _build_and_push_image( bad_json_format: bool = False, app_settings: ApplicationSettings, ) -> ServiceInRegistryInfoDict: - # crate image service_description = _create_service_description(service_type, name, tag) docker_labels = _create_docker_labels( @@ -214,12 +213,13 @@ async def _build_and_push_image( ) -def _clean_registry(registry_url: str, list_of_images: list[ServiceInRegistryInfoDict]): +def _clean_registry(list_of_images: list[ServiceInRegistryInfoDict]): request_headers = {"accept": "application/vnd.docker.distribution.manifest.v2+json"} for image in list_of_images: service_description = image["service_description"] # get the image digest tag = service_description["version"] + registry_url = image["image_path"].split("/")[0] url = "http://{host}/v2/{name}/manifests/{tag}".format( host=registry_url, name=service_description["key"], tag=tag ) @@ -243,15 +243,13 @@ async def __call__( inter_dependent_services: bool = False, bad_json_format: bool = False, version="1.0.", - ) -> list[ServiceInRegistryInfoDict]: - ... + ) -> list[ServiceInRegistryInfoDict]: ... @pytest.fixture def push_services( docker_registry: str, app_settings: ApplicationSettings ) -> Iterator[PushServicesCallable]: - registry_url = docker_registry list_of_pushed_images_tags: list[ServiceInRegistryInfoDict] = [] dependent_images = [] @@ -262,8 +260,13 @@ async def _build_push_images_to_docker_registry( inter_dependent_services=False, bad_json_format=False, version="1.0.", + override_registry_url: str | None = None, ) -> list[ServiceInRegistryInfoDict]: try: + registry_url = docker_registry + if override_registry_url: + _logger.info("Overriding registry URL with %s", override_registry_url) + registry_url = override_registry_url dependent_image = None if inter_dependent_services: dependent_image = await _build_and_push_image( @@ -317,5 +320,5 @@ async def _build_push_images_to_docker_registry( yield _build_push_images_to_docker_registry _logger.info("clean registry") - _clean_registry(registry_url, list_of_pushed_images_tags) - _clean_registry(registry_url, dependent_images) + _clean_registry(list_of_pushed_images_tags) + _clean_registry(dependent_images) diff --git a/services/director/tests/unit/test_registry_proxy.py b/services/director/tests/unit/test_registry_proxy.py index d4cb4d63c979..74e7c5b7ef34 100644 --- a/services/director/tests/unit/test_registry_proxy.py +++ b/services/director/tests/unit/test_registry_proxy.py @@ -21,7 +21,6 @@ async def test_list_no_services_available( configure_registry_access: EnvVarsDict, app: FastAPI, ): - computational_services = await registry_proxy.list_services( app, registry_proxy.ServiceType.COMPUTATIONAL ) @@ -116,13 +115,36 @@ async def test_list_interactive_service_dependencies( assert image_dependencies[0]["tag"] == docker_dependencies[0]["tag"] +@pytest.fixture( + params=["docker_registry", "docker_registry_v2"], ids=["registry_v3", "registry_v2"] +) +def configure_registry_access_both_versions( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, +) -> EnvVarsDict: + """Parametrized fixture that tests with both registry v3 and v2 - use only for specific tests that need both""" + registry_url = request.getfixturevalue(request.param) + return app_environment | setenvs_from_dict( + monkeypatch, + envs={ + "REGISTRY_URL": registry_url, + "REGISTRY_PATH": registry_url, + "REGISTRY_SSL": False, + "DIRECTOR_REGISTRY_CACHING": False, + }, + ) + + async def test_get_image_labels( - configure_registry_access: EnvVarsDict, + configure_registry_access_both_versions: EnvVarsDict, app: FastAPI, push_services, ): images = await push_services( - number_of_computational_services=1, number_of_interactive_services=1 + number_of_computational_services=1, + number_of_interactive_services=1, + override_registry_url=configure_registry_access_both_versions["REGISTRY_URL"], ) images_digests = set() for image in images: