Fix & unify helm test setup as make rules (#2867)

* Fix & unify helm test setup as make rules

* Allow to customize the test registry

* Skip Ci kind to use makefile kind cluster instead

Signed-off-by: jose.vazquez <[email protected]>

---------

Signed-off-by: jose.vazquez <[email protected]>

Author: josvazg

.github/workflows/test-e2e-gov.yml

@@ -14,20 +14,10 @@ jobs:
         uses: actions/checkout@v5
         with:
           submodules: true
-      - name: Create k8s Kind Cluster
-        if: ${{ !env.ACT }}
-        uses: helm/[email protected]
-        with:
-          version: v0.29.0
-          config: test/helper/e2e/config/kind.yaml
-          cluster_name: "atlas-gov-e2e-test"
-          wait: 180s
       - name: Install devbox
         uses: jetify-com/[email protected]
         with:
           enable-cache: 'true'
-      - name: Install CRDs
-        run: devbox run -- 'make install'
       - name: Run e2e test
         env:
           MCLI_PUBLIC_API_KEY: ${{ secrets.ATLAS_GOV_PUBLIC_KEY }}
@@ -39,7 +29,8 @@ jobs:
           AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
           PAGER_DUTY_SERVICE_KEY: ${{ secrets.PAGER_DUTY_SERVICE_KEY }}
           TEST_NAME: "atlas-gov"
-        run: devbox run -- ./scripts/launch-ci-e2e.sh
+          USE_NEXT_VERSION: "true"
+        run: devbox run -- make e2e label=${{ env.TEST_NAME}}
       - name: Upload operator logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v5

.github/workflows/test-e2e.yml

@@ -133,31 +133,14 @@ jobs:
         run: |
           devbox run -- make bundle
 
-
       - name: Extract k8s version/platform
         id: extract
         run: |
           echo "k8s_version=$(echo '${{ matrix.k8s }}' | awk -F '-' '{print $1}')" >> $GITHUB_OUTPUT
           echo "k8s_platform=$(echo '${{ matrix.k8s }}' | awk -F '-' '{print $2}')" >> $GITHUB_OUTPUT
 
-      - name: Setup kind cluster
-        if: ${{ steps.extract.outputs.k8s_platform == 'kind' }}
-        uses: helm/[email protected]
-        with:
-          version: v0.29.0
-          config: test/helper/e2e/config/kind.yaml
-          node_image: kindest/node:${{ steps.extract.outputs.k8s_version }}
-          cluster_name: ${{ matrix.test }}-${{ matrix.k8s }}
-          wait: 180s
-
-      - name: Print Kubernetes version
-        run: devbox run -- kubectl version
-
-      - name: Apply CRDs
-        run: devbox run -- make install-crds
-
       - name: Run CI E2E tests
-        run: devbox run -- ./scripts/launch-ci-e2e.sh
+        run: devbox run -- make e2e label=${{ env.TEST_NAME}}
         env:
           TEST_NAME: ${{ matrix.test }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -210,21 +193,8 @@ jobs:
           echo "k8s_version=$(echo '${{ matrix.k8s }}' | awk -F '-' '{print $1}')" >> $GITHUB_OUTPUT
           echo "k8s_platform=$(echo '${{ matrix.k8s }}' | awk -F '-' '{print $2}')" >> $GITHUB_OUTPUT
 
-      - name: Setup kind cluster
-        if: ${{ steps.extract.outputs.k8s_platform == 'kind' }}
-        uses: helm/[email protected]
-        with:
-          version: v0.29.0
-          config: test/helper/e2e/config/kind.yaml
-          node_image: kindest/node:${{ steps.extract.outputs.k8s_version }}
-          cluster_name: ${{ matrix.test }}-${{ matrix.k8s }}
-          wait: 180s
-
-      - name: Print Kubernetes version
-        run: devbox run -- kubectl version
-
       - name: Run CI helm-E2E test with prepared image
-        run: devbox run -- ./scripts/launch-ci-e2e.sh
+        run: devbox run -- make e2e label=${{ env.TEST_NAME }}
         env:
           TEST_NAME: ${{ matrix.test }}
           IMAGE_PULL_SECRET_REGISTRY: ghcr.io

.github/workflows/tests-e2e2.yaml

@@ -150,37 +150,17 @@ jobs:
           echo "k8s_version=$version" >> $GITHUB_OUTPUT
           echo "k8s_platform=$platform" >> $GITHUB_OUTPUT
 
-      - name: Create k8s Kind Cluster
-        if: ${{ steps.properties.outputs.k8s_platform == 'kind' && !env.ACT }}
-        uses: helm/[email protected]
-        with:
-          version: v0.29.0
-          config: test/helper/e2e/config/kind.yaml
-          node_image: kindest/node:${{ steps.properties.outputs.k8s_version }}
-          cluster_name: ${{ matrix.test }}
-          wait: 180s
-
-      - name: Print kubectl version
-        run: |
-          devbox run -- 'kubectl version'
-
-      - name: Install CRDs if needed
-        run: |
-          devbox run -- 'make install-crds'
-          devbox run -- 'make bundle'
-
       - name: Run E2E2 test
         env:
           MCLI_PUBLIC_API_KEY: ${{ secrets.ATLAS_PUBLIC_KEY }}
           MCLI_PRIVATE_API_KEY: ${{ secrets.ATLAS_PRIVATE_KEY }}
           MCLI_ORG_ID: ${{ secrets.ATLAS_ORG_ID}}
           MCLI_OPS_MANAGER_URL: "https://cloud-qa.mongodb.com/"
           TEST_NAME: "${{ matrix.test }}"
-          USE_KIND: false
+          USE_CURRENT_VERSION: "true"
         run: |
           echo "Using ENV: ${{ steps.select-env.outputs.ENV }}"
-          label=${TEST_NAME}
-          devbox run -- make e2e2 label="${label}"          
+          devbox run -- make e2e2 label=${{ env.TEST_NAME }}
 
       - name: Upload operator logs
         if: ${{ failure() }}

.github/workflows/tests-selectable.yaml

@@ -233,29 +233,12 @@ jobs:
           platform=$(echo ${{ matrix.k8s }} | awk -F "-" '{print $2}')
           echo "k8s_version=$version" >> $GITHUB_OUTPUT
           echo "k8s_platform=$platform" >> $GITHUB_OUTPUT
-      - name: Create k8s Kind Cluster
-        if: ${{ steps.properties.outputs.k8s_platform == 'kind' && !env.ACT }}
-        uses: helm/[email protected]
-        with:
-          version: v0.29.0
-          config: test/helper/e2e/config/kind.yaml
-          node_image: kindest/node:${{ steps.properties.outputs.k8s_version }}
-          cluster_name: ${{ matrix.test }}
-          wait: 180s
-      - name: Print kubectl version
-        run: |
-          devbox run -- 'kubectl version'
-      - name: Install CRDs if needed
-        if: ${{ !( matrix.test == 'helm-update' || matrix.test == 'helm-wide' || matrix.test == 'helm-ns' || matrix.test == 'bundle-test' ) }}
-        run: |
-          devbox run -- 'make install'
       - name: Run E2E test
         env:
           MCLI_OPS_MANAGER_URL: "https://cloud-qa.mongodb.com/"
           MCLI_ORG_ID: ${{ secrets.ATLAS_ORG_ID}}
           MCLI_PUBLIC_API_KEY: ${{ secrets.ATLAS_PUBLIC_KEY }}
           MCLI_PRIVATE_API_KEY: ${{ secrets.ATLAS_PRIVATE_KEY }}
-          IMAGE_URL: "ghcr.io/mongodb/mongodb-atlas-kubernetes-operator-prerelease:${{ needs.detect-tests.outputs.image_tag }}"
           BUNDLE_IMAGE: "ghcr.io/mongodb/mongodb-atlas-kubernetes-bundles-prerelease:${{ needs.detect-tests.outputs.image_tag }}"
           IMAGE_PULL_SECRET_REGISTRY: ghcr.io
           IMAGE_PULL_SECRET_USERNAME: $
@@ -273,7 +256,7 @@ jobs:
           PAGER_DUTY_SERVICE_KEY: ${{ secrets.PAGER_DUTY_SERVICE_KEY }}
         run: |
           echo "Using ENV: ${{ steps.select-env.outputs.ENV }}"
-          devbox run -- ./scripts/launch-ci-e2e.sh
+          devbox run -- make e2e label=${{ env.TEST_NAME }}
       - name: Upload operator logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v5
@@ -305,20 +288,10 @@ jobs:
           ref: ${{github.event.pull_request.head.sha}}
           submodules: true
           fetch-depth: 0
-      - name: Create k8s Kind Cluster
-        if: ${{ !env.ACT }}
-        uses: helm/[email protected]
-        with:
-          version: v0.29.0
-          config: test/helper/e2e/config/kind.yaml
-          cluster_name: "atlas-gov-e2e-test"
-          wait: 180s
       - name: Install devbox
         uses: jetify-com/[email protected]
         with:
           enable-cache: 'true'
-      - name: Install CRDs
-        run: devbox run -- 'make install'
       - name: Run e2e test
         env:
           MCLI_PUBLIC_API_KEY: ${{ secrets.ATLAS_GOV_PUBLIC_KEY }}
@@ -330,7 +303,7 @@ jobs:
           AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
           PAGER_DUTY_SERVICE_KEY: ${{ secrets.PAGER_DUTY_SERVICE_KEY }}
           TEST_NAME: "${{ matrix.test }}"
-        run: devbox run -- ./scripts/launch-ci-e2e.sh
+        run: devbox run -- make e2e label=${{ env.TEST_NAME }}
       - name: Upload operator logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v5

.gitignore

@@ -61,3 +61,6 @@ config/manifests/kustomization.yaml
 config/manager/kustomization.yaml
 config/generated/crd/bases/*.yaml
 config/generated/rbac/role.yaml
+
+# ignore helm test artifacts
+helm-charts/atlas-operator/charts/

Makefile

@@ -18,10 +18,21 @@ DOCKER_SBOM_PLUGIN_VERSION=0.6.1
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION_FILE=version.json
 CURRENT_VERSION := $(shell $(JQ) -r .current $(VERSION_FILE))
+NEXT_VERSION := $(shell $(JQ) -r .next $(VERSION_FILE))
 VERSION ?= $(shell git describe --always --tags --dirty --broken | cut -c 2-)
 BUILDTIME ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 GITCOMMIT ?= $(shell git rev-parse --short HEAD 2> /dev/null || true)
 
+# Fix for e2e-gov tests not to use a bad semver version instead
+ifdef USE_NEXT_VERSION
+VERSION=$(NEXT_VERSION)
+endif
+
+# Fix for e2e2 all-in-one test so that it uses an image that already exists in pre-release
+ifdef USE_CURRENT_VERSION
+VERSION=$(CURRENT_VERSION)
+endif
+
 VERSION_PACKAGE = github.com/mongodb/mongodb-atlas-kubernetes/v2/internal/version
 
 # LD_FLAGS
@@ -32,13 +43,6 @@ ifdef EXPERIMENTAL
 LD_FLAGS += -X $(VERSION_PACKAGE).Experimental=$(EXPERIMENTAL)
 endif
 
-# NEXT_VERSION represents a version that is higher than anything released
-# VERSION default value does not play well with the run target which might end up failing
-# with errors such as:
-# "version of the resource $Resource is higher than the operator version $VERSION"
-# This happens if you use exported YAMLs from CLI and the dirty version is deemed a pre-release
-NEXT_VERSION = 99.99.99-next
-
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable")
 # To re-generate a bundle for other specific channels without changing the standard setup, you can:
@@ -188,6 +192,18 @@ SBOMS_DIR ?= template
 
 SHELLCHECK_OPTIONS ?= -e SC2086
 
+TEST_REGISTRY ?= localhost:5000
+DEFAULT_IMAGE_URL := $(TEST_REGISTRY)/mongodb-atlas-kubernetes-operator:$(NEXT_VERSION)-test
+export IMAGE_URL
+
+ifndef IMAGE_URL
+    IMAGE_URL := $(DEFAULT_IMAGE_URL)
+	BUILD_DEPENDENCY := test-docker-image
+else
+    $(info --- IMAGE_URL is set externally: $(IMAGE_URL))
+    BUILD_DEPENDENCY :=
+endif
+
 .DEFAULT_GOAL := help
 .PHONY: help
 help: ## Show this help screen
@@ -246,11 +262,11 @@ envtest-assets:
 	mkdir -p $(ENVTEST_ASSETS_DIR)
 
 .PHONY: e2e
-e2e: bundle manifests run-kind ## Run e2e test. Command `make e2e label=cluster-ns` run cluster-ns test
+e2e: bundle manifests run-kind install-crds $(BUILD_DEPENDENCY) ## Run e2e test. Command `make e2e label=cluster-ns` run cluster-ns test
 	AKO_E2E_TEST=1 $(GINKGO) $(shell pwd)/test/$@
 
 .PHONY: e2e2
-e2e2: run-kind manager install-credentials install-crds set-namespace ## Run e2e2 tests. Command `make e2e2 label=integrations-ctlr` run integrations-ctlr e2e2 test
+e2e2: bundle run-kind manager install-credentials install-crds set-namespace ## Run e2e2 tests. Command `make e2e2 label=integrations-ctlr` run integrations-ctlr e2e2 test
 	NO_GORUN=1 \
 	AKO_E2E2_TEST=1 \
 	OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) \
@@ -485,8 +501,21 @@ all-platforms:
 
 .PHONY: all-platforms-docker
 all-platforms-docker: all-platforms
-	docker build --build-arg BINARY_PATH=bin/linux/amd64 -f fast.Dockerfile -t manager-amd64 .
-	docker build --build-arg BINARY_PATH=bin/linux/arm64 -f fast.Dockerfile -t manager-arm64 .
+	docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=amd64 \
+	-f fast.Dockerfile -t manager-amd64 .
+	docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=arm64 \
+	-f fast.Dockerfile -t manager-arm64 .
+
+# docker-image builds the test image always for linux, even on MacOS.
+# This is because the Kubernetes cluster is always run within a Linux VM
+.PHONY: docker-image
+docker-image: all-platforms
+	docker build --build-arg TARGETOS=linux --build-arg TARGETARCH=$(TARGET_ARCH) \
+	-f fast.Dockerfile -t $(DEFAULT_IMAGE_URL) .
+
+.PHONY: test-docker-image
+test-docker-image: docker-image run-kind
+	docker push $(DEFAULT_IMAGE_URL)
 
 .PHONY: check-major-version
 check-major-version: ## Check that VERSION starts with MAJOR_VERSION