🌱 fix e2e caph image. (#1561)

guettli · web-flow · commit 0a53616ccd28 · 2025-03-19T11:33:37.000+01:00
diff --git a/.github/actions/e2e/action.yaml b/.github/actions/e2e/action.yaml
@@ -7,9 +7,6 @@ inputs:
   e2e_make_target:
     description: "e2e_make_target"
     required: true
-  e2e_hcloud_token:
-    description: "e2e_hcloud_token"
-    required: false
   e2e_robot_user:
     description: "e2e_robot_user"
     required: false
@@ -57,10 +54,8 @@ runs:
       env:
         TAG: ${{ steps.meta.outputs.version }}
 
-    - name: HCLOUD_TOKEN
+    - name: Get HCLOUD_TOKEN # sets env.HCLOUD_TOKEN
       uses: hetznercloud/tps-action@dee5dd2546322c28ed8f74b910189066e8b6f31a # main
-      with:
-        token: ${{ inputs.e2e_hcloud_token }}
 
     - name: "e2e-${{ inputs.e2e_name }}"
       shell: bash
@@ -69,7 +64,6 @@ runs:
         REGISTRY: ${{ env.REGISTRY }}
         IMAGE_NAME: ${{ env.IMAGE_NAME }}
         TAG: ${{ steps.meta.outputs.version }}
-        MANIFEST_PATH: "../../../out"
         HCLOUD_TOKEN: ${{ env.HCLOUD_TOKEN }}
         HETZNER_ROBOT_USER: ${{ inputs.e2e_robot_user }}
         HETZNER_ROBOT_PASSWORD: ${{ inputs.e2e_robot_password }}
diff --git a/.github/workflows/e2e-basic-baremetal.yaml b/.github/workflows/e2e-basic-baremetal.yaml
@@ -47,7 +47,6 @@ jobs:
         with:
           e2e_name: hetzner-basic
           e2e_make_target: test-e2e-baremetal
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
           e2e_robot_user: ${{ secrets.HETZNER_ROBOT_USER }}
           e2e_robot_password: ${{ secrets.HETZNER_ROBOT_PASSWORD }}
           e2e_ssh_pub: ${{ secrets.HETZNER_SSH_PUB }}
diff --git a/.github/workflows/e2e-basic.yaml b/.github/workflows/e2e-basic.yaml
@@ -47,4 +47,3 @@ jobs:
         with:
           e2e_name: hcloud-basic
           e2e_make_target: test-e2e-hcloud
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
diff --git a/.github/workflows/e2e-feature-baremetal.yaml b/.github/workflows/e2e-feature-baremetal.yaml
@@ -47,7 +47,6 @@ jobs:
         with:
           e2e_name: hetzner-baremetal-feature
           e2e_make_target: test-e2e-baremetal-feature
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
           e2e_robot_user: ${{ secrets.HETZNER_ROBOT_USER }}
           e2e_robot_password: ${{ secrets.HETZNER_ROBOT_PASSWORD }}
           e2e_ssh_pub: ${{ secrets.HETZNER_SSH_PUB }}
diff --git a/.github/workflows/e2e-feature.yaml b/.github/workflows/e2e-feature.yaml
@@ -47,4 +47,3 @@ jobs:
         with:
           e2e_name: hcloud-feature
           e2e_make_target: test-e2e-feature
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
diff --git a/.github/workflows/e2e-periodic.yaml b/.github/workflows/e2e-periodic.yaml
@@ -47,7 +47,6 @@ jobs:
         with:
           e2e_name: hcloud-basic
           e2e_make_target: test-e2e-hcloud
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
 
   e2e-hcloud-feature:
     name: Test Hcloud Features
@@ -68,7 +67,7 @@ jobs:
         with:
           e2e_name: hcloud-feature
           e2e_make_target: test-e2e-feature
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
+
   e2e-hcloud-lifecycle:
     name: Test Hcloud Lifecycle
     concurrency: ci-${{ github.ref }}-e2e-lifecycle
@@ -88,7 +87,6 @@ jobs:
         with:
           e2e_name: hcloud-lifecycle
           e2e_make_target: test-e2e-lifecycle
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
 
   e2e-hcloud-conformance:
     name: Test Hcloud Conformance
@@ -109,4 +107,3 @@ jobs:
         with:
           e2e_name: hcloud-conformance
           e2e_make_target: test-e2e-conformance
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
diff --git a/.github/workflows/e2e-upgrade-caph.yaml b/.github/workflows/e2e-upgrade-caph.yaml
@@ -47,7 +47,6 @@ jobs:
         with:
           e2e_name: upgrade-caph
           e2e_make_target: test-e2e-upgrade-caph
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
           e2e_robot_user: ${{ secrets.HETZNER_ROBOT_USER }}
           e2e_robot_password: ${{ secrets.HETZNER_ROBOT_PASSWORD }}
           e2e_ssh_pub: ${{ secrets.HETZNER_SSH_PUB }}
diff --git a/.github/workflows/e2e-upgrade-kubernetes.yaml b/.github/workflows/e2e-upgrade-kubernetes.yaml
@@ -47,4 +47,3 @@ jobs:
         with:
           e2e_name: upgrade-kubernetes
           e2e_make_target: test-e2e-upgrade-kubernetes
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
diff --git a/.github/workflows/pr-e2e.yaml b/.github/workflows/pr-e2e.yaml
@@ -59,7 +59,6 @@ jobs:
         with:
           e2e_name: hcloud-basic
           e2e_make_target: test-e2e-hcloud
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
 
   e2e-hetzner-basic:
     name: Test Hetzner Baremetal Basic
@@ -81,7 +80,6 @@ jobs:
         with:
           e2e_name: hetzner-basic
           e2e_make_target: test-e2e-baremetal
-          e2e_hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
           e2e_robot_user: ${{ secrets.HETZNER_ROBOT_USER }}
           e2e_robot_password: ${{ secrets.HETZNER_ROBOT_PASSWORD }}
           e2e_ssh_pub: ${{ secrets.HETZNER_SSH_PUB }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -67,6 +67,12 @@ jobs:
         with:
           paths: ".coverage/junit.xml"
 
+      - name: Check Release Manifests
+        shell: bash
+        run: |
+          make release-manifests
+          ./hack/check-release-manifests.sh
+
       - name: Upload Report
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         if: ${{ !cancelled() }}
diff --git a/Makefile b/Makefile
@@ -57,7 +57,6 @@ help: ## Display this help.
 BUILD_IN_CONTAINER ?= true
 
 # Boiler plate for building Docker containers.
-TAG ?= dev
 ARCH ?= amd64
 # Allow overriding the imagePullPolicy
 PULL_POLICY ?= Always
@@ -177,10 +176,6 @@ install-crds: generate-manifests $(KUSTOMIZE) ## Install CRDs into the K8s clust
 uninstall-crds: generate-manifests $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete -f -
 
-deploy-controller: generate-manifests $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMAGE_PREFIX}/$(STAGING_IMAGE):${TAG}
-	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
-
 undeploy-controller: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/default | $(KUBECTL) delete -f -
 
@@ -369,6 +364,8 @@ $(RELEASE_NOTES_DIR):
 
 .PHONY: test-release
 test-release:
+	@# TAG: caph container image tag. For PRs this is pr-NNNN
+	./hack/ensure-env-variables.sh TAG
 	$(MAKE) set-manifest-image MANIFEST_IMG=$(IMAGE_PREFIX)/$(STAGING_IMAGE) MANIFEST_TAG=$(TAG)
 	$(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent
 	$(MAKE) release-manifests
@@ -391,6 +388,8 @@ release: clean-release  ## Builds and push container images using the latest git
 	$(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent
 	## Build the manifests
 	$(MAKE) release-manifests clean-release-git
+	./hack/check-release-manifests.sh
+
 
 .PHONY: release-notes
 release-notes: $(RELEASE_NOTES_DIR) $(RELEASE_NOTES)
@@ -471,13 +470,16 @@ test-unit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests
 
 .PHONY: e2e-image
 e2e-image: ## Build the e2e manager image
-	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -t $(IMAGE_PREFIX)/$(STAGING_IMAGE):e2e -f images/$(INFRA_SHORT)/Dockerfile .
+	./hack/ensure-env-variables.sh TAG
+	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -t $(IMAGE_PREFIX)/$(STAGING_IMAGE):$(TAG) -f images/$(INFRA_SHORT)/Dockerfile .
 
 .PHONY: e2e-conf-file
 e2e-conf-file: $(E2E_CONF_FILE)
 $(E2E_CONF_FILE): $(ENVSUBST) $(E2E_CONF_FILE_SOURCE) ./hack/create-e2e-conf-file.sh
-	CAPH_LATEST_VERSION=$(CAPH_LATEST_VERSION) ENVSUBST=$(ENVSUBST) E2E_CONF_FILE_SOURCE=$(E2E_CONF_FILE_SOURCE) \
-		E2E_CONF_FILE=$(E2E_CONF_FILE) ./hack/create-e2e-conf-file.sh
+	CAPH_LATEST_VERSION=$(CAPH_LATEST_VERSION) ENVSUBST=$(ENVSUBST) \
+	E2E_CONF_FILE_SOURCE=$(E2E_CONF_FILE_SOURCE) \
+	E2E_CONF_FILE=$(E2E_CONF_FILE) \
+	./hack/create-e2e-conf-file.sh
 
 .PHONY: test-e2e
 test-e2e: test-e2e-hcloud
@@ -488,8 +490,8 @@ test-e2e-hcloud: $(E2E_CONF_FILE) $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFAC
 	HETZNER_SSH_PUB= HETZNER_SSH_PRIV= \
 	HETZNER_SSH_PUB_PATH= HETZNER_SSH_PRIV_PATH= \
 	HETZNER_ROBOT_PASSWORD= HETZNER_ROBOT_USER= \
-	GINKGO_FOKUS="'\[Basic\]'" GINKGO_NODES=2 E2E_CONF_FILE=$(E2E_CONF_FILE) \
-		./hack/ci-e2e-capi.sh
+	GINKGO_FOKUS="'\[Basic\]'" GINKGO_NODES=2 \
+	./hack/ci-e2e-capi.sh
 
 .PHONY: test-e2e-feature
 test-e2e-feature: $(E2E_CONF_FILE) $(if $(SKIP_IMAGE_BUILD),,e2e-image) $(ARTIFACTS)
diff --git a/controllers/hetznercluster_controller_test.go b/controllers/hetznercluster_controller_test.go
@@ -558,7 +558,24 @@ var _ = Describe("Hetzner ClusterReconciler", func() {
 				By("checking that cluster is ready")
 
 				Eventually(func() bool {
-					return isPresentAndTrue(key, instance, infrav1.LoadBalancerReadyCondition)
+					err := testEnv.Get(ctx, client.ObjectKeyFromObject(instance), instance)
+					if err != nil {
+						return false
+					}
+					c := conditions.Get(instance, infrav1.LoadBalancerReadyCondition)
+					if c == nil {
+						GinkgoLogr.Info("LoadBalancerReadyCondition is nil")
+						return false
+					}
+					if c.Status == corev1.ConditionTrue {
+						GinkgoLogr.Info("LoadBalancerReadyCondition is True now")
+						return true
+					}
+					GinkgoLogr.Info("LoadBalancerReadyCondition is not True yet.",
+						"reason", c.Reason,
+						"message", c.Message,
+					)
+					return false
 				}, 2*timeout, time.Second).Should(BeTrue()) // flaky ?
 
 				By("checking that load balancer has label set")
diff --git a/hack/check-release-manifests.sh b/hack/check-release-manifests.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bash Strict Mode: https://github.com/guettli/bash-strict-mode
+trap 'echo "Warning: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0")"; exit 3' ERR
+set -Eeuo pipefail
+
+if grep :pr- out/*yaml; then
+    echo "Error: ':pr-' found in out/*yaml. This looks like a PR container image tag."
+    exit 1
+fi
diff --git a/hack/ci-e2e-capi.sh b/hack/ci-e2e-capi.sh
@@ -14,10 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -o errexit
-set -o pipefail
+trap 'echo "Warning: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0")"; exit 3' ERR
+set -Eeuo pipefail
 
-REPO_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
+REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")
 cd "${REPO_ROOT}" || exit 1
 
 # Make sure the tools binaries are on the path.
@@ -40,11 +40,18 @@ trap 'remove_ssh_key ${SSH_KEY_NAME}' EXIT
 mkdir -p "$ARTIFACTS"
 echo "+ run tests!"
 
-if [[ "${CI:-""}" == "true" ]]; then
-    make set-manifest-image MANIFEST_IMG=${IMAGE_PREFIX}/caph-staging MANIFEST_TAG=${TAG}
-    make set-manifest-pull-policy PULL_POLICY=IfNotPresent
+IMAGE_PREFIX="${IMAGE_PREFIX:-ghcr.io/syself}"
+
+if [[ -z "${TAG:-}" ]]; then
+    echo
+    echo "Error: Missing TAG environment variable"
+    echo "This is the caph container image tag for the image."
+    echo "For PRs this is pr-NNNN"
+    echo "Use the following command to set the environment variable:"
+    echo "  gh pr view --json number --jq .number"
+    echo "Then: export TAG=pr-NNNN"
+    echo
+    exit 1
 fi
 
 make -C test/e2e/ run GINKGO_NODES="${GINKGO_NODES}" GINKGO_FOCUS="${GINKGO_FOKUS}"
-
-test_status="${?}"
diff --git a/hack/create-e2e-conf-file.sh b/hack/create-e2e-conf-file.sh
@@ -17,13 +17,25 @@
 trap 'echo "ERROR: A command has failed. Exiting the script. Line was ($0:$LINENO): $(sed -n "${LINENO}p" "$0")"; exit 3' ERR
 set -Eeuo pipefail
 
-./hack/ensure-env-variables.sh CAPH_LATEST_VERSION ENVSUBST E2E_CONF_FILE_SOURCE E2E_CONF_FILE
+./hack/ensure-env-variables.sh CAPH_LATEST_VERSION ENVSUBST E2E_CONF_FILE_SOURCE E2E_CONF_FILE TAG
 
-# When called via CI, MANIFEST_PATH is already set. It points to the "out" directory
-# created with the Github action "download-artifacts".
-if [ -z "${MANIFEST_PATH:-}" ]; then
-    export MANIFEST_PATH="../../.."
+if [[ -z "${TAG:-}" ]]; then
+    echo
+    echo "Error: Missing TAG environment variable"
+    echo "This is the caph container image tag for the image."
+    echo "For PRs this is pr-NNNN"
+    echo "Use the following command to set the environment variable:"
+    echo "  gh pr view --json number --jq .number"
+    echo "Then: export TAG=pr-NNNN"
+    echo
+    exit 1
 fi
 
+make release-manifests
+
+# `make release-manifests` changes local files (caph image). Restore them,
+# so they do not get committed accidentally.
+git restore config
+
 echo "# Created from $E2E_CONF_FILE_SOURCE by $0" >"$E2E_CONF_FILE"
 $ENVSUBST <"$E2E_CONF_FILE_SOURCE" >>"$E2E_CONF_FILE"
diff --git a/test/e2e/config/hetzner.yaml b/test/e2e/config/hetzner.yaml
@@ -8,7 +8,7 @@
 
 # For creating local dev images run make e2e-image from the main CAPH repository
 images:
-  - name: ghcr.io/syself/caph-staging:${TAG:=e2e}
+  - name: ghcr.io/syself/caph-staging:${TAG}
     loadBehavior: mustLoad
 providers:
   - name: cluster-api
@@ -69,7 +69,31 @@ providers:
         type: "url"
         contract: v1beta1
         files:
-          - sourcePath: "${MANIFEST_PATH}/metadata.yaml"
+          - sourcePath: "../../../out/metadata.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-k8s-upgrade.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-k8s-upgrade-kcp-scale-in.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-hcloud-feature-load-balancer-extra-services.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-hcloud-feature-placement-groups.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-hcloud-feature-loadbalancer-off.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-network.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-node-drain.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-md-remediation.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-kcp-remediation.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-hetzner-baremetal.yaml"
+          - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-hetzner-baremetal-feature-raid-setup.yaml"
+      - name: v1.0.99 # next; use manifest from source files
+        value: ../../../config/default
+        contract: v1beta1
+        replacements:
+          - old: ghcr.io/syself/caph-staging:latest
+            new: ghcr.io/syself/caph-staging:${TAG}
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+          - old: --metrics-bind-addr=localhost:8080
+            new: --metrics-bind-addr=:8080
+        files:
+          - sourcePath: "../../../metadata.yaml"
           - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template.yaml"
           - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-k8s-upgrade.yaml"
           - sourcePath: "../data/infrastructure-hetzner/v1beta1/cluster-template-k8s-upgrade-kcp-scale-in.yaml"
@@ -89,7 +113,7 @@ providers:
       #   contract: v1beta1
       #   replacements:
       #     - old: ghcr.io/syself/caph-staging:latest
-      #       new: ghcr.io/syself/caph-staging:e2e
+      #       new: ghcr.io/syself/caph-staging:${TAG}
       #     - old: "imagePullPolicy: Always"
       #       new: "imagePullPolicy: IfNotPresent"
       #     - old: --metrics-bind-addr=localhost:8080
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
