ci: Make terraform and orchestratord tests depend on devel tags

Author: def-

ci/mkpipeline.py

@@ -624,6 +624,7 @@ def trim_test_selection_id(pipeline: Any, step_ids_to_run: set[int]) -> None:
                 "build-aarch64",
                 "build-x86_64-lto",
                 "build-aarch64-lto",
+                "devel-docker-tags",
             )
             and not step.get("async")
         ):
@@ -646,6 +647,7 @@ def trim_test_selection_name(pipeline: Any, steps_to_run: set[str]) -> None:
                 "build-aarch64",
                 "build-x86_64-lto",
                 "build-aarch64-lto",
+                "devel-docker-tags",
             )
             and not step.get("async")
         ):
@@ -921,6 +923,11 @@ def trim_builds(
         elif step.get("id") == "upload-debug-symbols-aarch64":
             if hash_check[Arch.AARCH64][1]:
                 step["skip"] = True
+        elif step.get("id") == "devel-docker-tags":
+            step["concurrency"] = 1
+            step["concurrency_group"] = (
+                f"devel-docker-tags/{hash_check[Arch.X86_64][0]}"
+            )
 
 
 _github_changed_files: set[str] | None = None

ci/nightly/pipeline.template.yml

@@ -74,6 +74,24 @@ steps:
           queue: hetzner-aarch64-16cpu-32gb
         branches: "main"
 
+      - id: devel-docker-tags
+        label: Tag development docker images
+        command: bin/ci-builder run min bin/pyactivate -m ci.test.dev_tag
+        inputs:
+          - "*"
+        depends_on:
+          - build-x86_64
+          - build-aarch64
+        timeout_in_minutes: 20
+        agents:
+          queue: linux-aarch64-small
+        coverage: skip
+        # Fortify against intermittent DockerHub/GHCR issues
+        retry:
+          automatic:
+            - exit_status: 1
+              limit: 2
+
   - group: Linters
     key: linters
     steps:
@@ -1442,7 +1460,7 @@ steps:
     - id: terraform-aws
       label: "Terraform + Helm Chart E2E on AWS"
       artifact_paths: [test/terraform/aws-temporary/terraform.tfstate, "mz_debug_*.zip"]
-      depends_on: build-aarch64
+      depends_on: devel-docker-tags
       timeout_in_minutes: 120
       concurrency: 1
       concurrency_group: 'terraform-aws'
@@ -1461,7 +1479,7 @@ steps:
     - id: terraform-aws-upgrade
       label: "Terraform + Helm Chart Upgrade on AWS"
       artifact_paths: [test/terraform/aws-upgrade/terraform.tfstate, "mz_debug_*.zip"]
-      depends_on: build-aarch64
+      depends_on: devel-docker-tags
       timeout_in_minutes: 120
       concurrency: 1
       concurrency_group: 'terraform-upgrade-aws'
@@ -1485,7 +1503,7 @@ steps:
     - id: terraform-gcp
       label: "Terraform + Helm Chart E2E on GCP"
       artifact_paths: [test/terraform/gcp-temporary/terraform.tfstate, "mz_debug_*.zip"]
-      depends_on: build-aarch64
+      depends_on: devel-docker-tags
       timeout_in_minutes: 120
       concurrency: 1
       concurrency_group: 'terraform-gcp'
@@ -1504,7 +1522,7 @@ steps:
     - id: terraform-azure
       label: "Terraform + Helm Chart E2E on Azure"
       artifact_paths: [test/terraform/azure-temporary/terraform.tfstate, "mz_debug_*.zip"]
-      depends_on: build-aarch64
+      depends_on: devel-docker-tags
       timeout_in_minutes: 120
       concurrency: 1
       concurrency_group: 'terraform-azure'
@@ -2346,7 +2364,7 @@ steps:
     steps:
       - id: orchestratord-defaults
         label: "Orchestratord test (defaults from documentation)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
@@ -2358,7 +2376,7 @@ steps:
 
       - id: orchestratord-default-properties
         label: "Orchestratord test (defaults for properties)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
@@ -2370,7 +2388,7 @@ steps:
 
       - id: orchestratord-individual
         label: "Orchestratord test (individual properties)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
@@ -2394,7 +2412,7 @@ steps:
 
       - id: orchestratord-upgrade-individual
         label: "Orchestratord test (upgrade, individual props)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
@@ -2409,7 +2427,7 @@ steps:
 
       - id: orchestratord-upgrade-combine
         label: "Orchestratord test (upgrade, combine props)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
@@ -2424,7 +2442,7 @@ steps:
 
       - id: orchestratord-upgrade-chain-individual
         label: "Orchestratord test (upgrade chain, individual props)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:
@@ -2439,7 +2457,7 @@ steps:
 
       - id: orchestratord-upgrade-chain-combine
         label: "Orchestratord test (upgrade chain, combine props)"
-        depends_on: build-aarch64
+        depends_on: devel-docker-tags
         timeout_in_minutes: 120
         plugins:
           - ./ci/plugins/mzcompose:

ci/test/pipeline.template.yml

@@ -140,7 +140,7 @@ steps:
         agents:
           queue: linux-aarch64-small
         coverage: skip
-        # Fortify against intermittent DockerHub issues
+        # Fortify against intermittent DockerHub/GHCR issues
         retry:
           automatic:
             - exit_status: 1

misc/python/materialize/mzbuild.py

@@ -1451,10 +1451,17 @@ def publish_multiarch_images(
         names = set(image.image.name for image in images)
         assert len(names) == 1, "dependency sets did not contain identical images"
         name = images[0].image.docker_name(tag)
-        spawn.runv(
-            ["docker", "manifest", "create", name, *(image.spec() for image in images)]
-        )
-        spawn.runv(["docker", "manifest", "push", name])
+        if not is_docker_image_pushed(name):
+            spawn.runv(
+                [
+                    "docker",
+                    "manifest",
+                    "create",
+                    name,
+                    *(image.spec() for image in images),
+                ]
+            )
+            spawn.runv(["docker", "manifest", "push", name])
     print(f"--- Nofifying for tag {tag}")
     markdown = f"""Pushed images with Docker tag `{tag}`"""
     spawn.runv(

test/terraform/mzcompose.py

@@ -687,7 +687,7 @@ def setup(
 
         vars = [
             "-var",
-            "operator_version={get_operator_version()}",
+            f"operator_version={get_operator_version()}",
         ]
         if orchestratord_override:
             vars += [
@@ -1170,7 +1170,7 @@ def workflow_gcp_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
 
         vars = [
             "-var",
-            "operator_version={get_operator_version()}",
+            f"operator_version={get_operator_version()}",
         ]
         if args.orchestratord_override:
             vars += [
@@ -1282,7 +1282,7 @@ def workflow_azure_temporary(c: Composition, parser: WorkflowArgumentParser) ->
 
         vars = [
             "-var",
-            "operator_version={get_operator_version()}",
+            f"operator_version={get_operator_version()}",
         ]
         if args.orchestratord_override:
             vars += [