stop using chart-testing

Author: diamonwiggins

.github/workflows/mlflow-ci.yml

@@ -38,9 +38,6 @@ jobs:
         with:
           python-version: 3.12
 
-      - name: Set up chart-testing
-        uses: helm/[email protected]
-
       - name: Add Helm repositories
         run: |
           cd applications/mlflow
@@ -170,9 +167,6 @@ jobs:
         with:
           python-version: 3.12
 
-      - name: Set up chart-testing
-        uses: helm/[email protected]
-
       # Install jq via apt-get
       - name: Install jq
         run: |
@@ -248,7 +242,7 @@ jobs:
         env:
           REPLICATED_LICENSE_ID: ${{ steps.get-license.outputs.license_id }}
 
-      - name: Run Helm installation test with chart-testing
+      - name: Run Helm installation test with custom values
         run: |
           cd applications/mlflow
           # Save kubeconfig to a file
@@ -267,7 +261,7 @@ jobs:
           
           echo "Running test '${{ matrix.config.name }}' with MLflow values file: $MLFLOW_VALUES"
           
-          # Run chart testing installation using our updated make target that uses 'ct'
+          # Run direct Helm installation
           make test-replicated-helm-with-values
         env:
           KUBECONFIG: ${{ steps.create-cluster.outputs.cluster-kubeconfig }}

applications/mlflow/Makefile

@@ -45,16 +45,14 @@ add-helm-repositories:
 	helm repo add minio https://operator.min.io/; \
 	helm repo update
 
-# Target to lint Helm charts using chart-testing
+# Target to lint Helm charts using direct Helm commands
 .PHONY: lint
 lint: add-helm-repositories
 	echo "Linting Helm charts..."; \
-	if [ -f ./ct.yaml ]; then \
-		ct lint --config ./ct.yaml --chart-dirs $(HELM_CHARTS_DIR) --charts $(HELM_CHARTS_DIR)/mlflow $(HELM_CHARTS_DIR)/infra; \
-	else \
-		echo "Warning: ct.yaml not found. Using default configuration."; \
-		ct lint --chart-dirs $(HELM_CHARTS_DIR) --charts $(HELM_CHARTS_DIR)/mlflow $(HELM_CHARTS_DIR)/infra; \
-	fi; \
+	for chart in $(CHARTS); do \
+		echo "Linting $$chart chart..."; \
+		helm lint $(HELM_CHARTS_DIR)/$$chart; \
+	done; \
 	echo "Linting completed successfully."
 
 # Target to template Helm charts (render templates without installing)
@@ -78,18 +76,27 @@ template: add-helm-repositories
 test: lint template
 	echo "Running full chart installation test..."; \
 	echo "Note: This requires a connected Kubernetes cluster."; \
-	if [ -f ./ct.yaml ]; then \
-		ct install --config ./ct.yaml --chart-dirs $(HELM_CHARTS_DIR) \
-			--charts $(HELM_CHARTS_DIR)/infra && \
-		ct install --config ./ct.yaml --chart-dirs $(HELM_CHARTS_DIR) \
-			--charts $(HELM_CHARTS_DIR)/mlflow --set replicated.enabled=false; \
-	else \
-		echo "Warning: ct.yaml not found. Using default configuration."; \
-		ct install --chart-dirs $(HELM_CHARTS_DIR) \
-			--charts $(HELM_CHARTS_DIR)/infra && \
-		ct install --chart-dirs $(HELM_CHARTS_DIR) \
-			--charts $(HELM_CHARTS_DIR)/mlflow --set replicated.enabled=false; \
-	fi; \
+	\
+	# Create namespace if it doesn't exist
+	kubectl create namespace mlflow-test 2>/dev/null || true; \
+	\
+	# Install infra chart
+	echo "Installing infra chart..."; \
+	helm upgrade --install infra-test $(HELM_CHARTS_DIR)/infra \
+		--namespace mlflow-test \
+		--wait --timeout 5m --debug; \
+	\
+	# Install MLflow chart with SDK disabled
+	echo "Installing mlflow chart..."; \
+	helm upgrade --install mlflow-test $(HELM_CHARTS_DIR)/mlflow \
+		--namespace mlflow-test \
+		--set replicated.enabled=false \
+		--wait --timeout 5m --debug; \
+	\
+	# Test the installation
+	echo "Verifying installation..."; \
+	kubectl get pods -n mlflow-test; \
+	\
 	echo "Full installation test completed successfully."
 
 # Target to login to the Replicated registry
@@ -105,77 +112,44 @@ registry-login:
 		--password="$$REPLICATED_LICENSE_ID"; \
 	echo "Registry login successful."
 
-# Target to test Helm installation with charts from Replicated registry
-.PHONY: test-replicated-helm
-test-replicated-helm: registry-login
-	echo "Running Helm installation test with charts from Replicated registry..."; \
-	echo "Note: This requires REPLICATED_APP and REPLICATED_CHANNEL env vars."; \
-	if [ -z "$$REPLICATED_APP" ] || [ -z "$$REPLICATED_CHANNEL" ]; then \
-		echo "ERROR: REPLICATED_APP and REPLICATED_CHANNEL must be set"; \
-		exit 1; \
-	fi; \
-	OCI_URL="oci://registry.replicated.com/$$REPLICATED_APP/$$REPLICATED_CHANNEL"; \
-	echo "Creating temporary ct-oci.yaml config file..."; \
-	echo "chart-repos:" > ct-oci.yaml; \
-	echo "  - replicated=$$OCI_URL" >> ct-oci.yaml; \
-	echo "debug: true" >> ct-oci.yaml; \
-	cat ct-oci.yaml; \
-	echo "Listing available charts in the registry..."; \
-	helm search repo replicated || true; \
-	echo "Attempting chart installation using chart-testing..."; \
-	if ct install --config ct-oci.yaml \
-		--charts "infra" \
-		--namespace default \
-		--release-label "ci-test" && \
-	   ct install --config ct-oci.yaml \
-		--charts "mlflow" \
-		--namespace default \
-		--release-label "ci-test"; then \
-		echo "Chart-testing installation successful"; \
-	else \
-		echo "Chart-testing install failed, falling back to direct helm install..."; \
-		helm install infra-direct $$OCI_URL/infra --wait --timeout 5m --debug --namespace default; \
-		helm install mlflow-direct $$OCI_URL/mlflow --wait --timeout 5m --debug --namespace default; \
-	fi; \
-	rm -f ct-oci.yaml; \
-	echo "Replicated Helm installation test completed successfully."
-
-# Target for testing with chart-testing (ct) and custom values
+# Target for testing with custom values
 .PHONY: test-replicated-helm-with-values
 test-replicated-helm-with-values: registry-login
-	echo "Running Helm installation test with custom values using chart-testing..."; \
+	echo "Running Helm installation test with custom values..."; \
 	echo "Note: This requires REPLICATED_APP and REPLICATED_CHANNEL env vars."; \
 	if [ -z "$$REPLICATED_APP" ] || [ -z "$$REPLICATED_CHANNEL" ]; then \
 		echo "ERROR: REPLICATED_APP and REPLICATED_CHANNEL must be set"; \
 		exit 1; \
 	fi; \
 	OCI_URL="oci://registry.replicated.com/$$REPLICATED_APP/$$REPLICATED_CHANNEL"; \
-	echo "Creating temporary ct-oci.yaml config file with custom values..."; \
-	echo "chart-repos:" > ct-oci.yaml; \
-	echo "  - replicated=$$OCI_URL" >> ct-oci.yaml; \
-	echo "debug: true" >> ct-oci.yaml; \
 	\
 	# Prepare values arguments if provided
+	MLFLOW_VALUES_ARGS=""; \
 	if [ -n "$$MLFLOW_VALUES" ]; then \
 		echo "Using MLflow values file: $$MLFLOW_VALUES"; \
-		echo "helm-extra-args: --values $$MLFLOW_VALUES" >> ct-oci.yaml; \
+		MLFLOW_VALUES_ARGS="--values $$MLFLOW_VALUES"; \
 	fi; \
 	\
-	cat ct-oci.yaml; \
+	# Create namespace if it doesn't exist
+	kubectl create namespace values-test 2>/dev/null || true; \
+	\
+	# Install infra chart from Replicated registry
+	echo "Installing infra chart from Replicated registry..."; \
+	helm upgrade --install infra-values-test $$OCI_URL/infra \
+		--namespace values-test \
+		--wait --timeout 5m --debug; \
 	\
-	echo "Installing infra chart with chart-testing..."; \
-	ct install --config ct-oci.yaml \
-		--charts "infra" \
-		--namespace default \
-		--release-label "ci-test"; \
+	# Install MLflow chart from Replicated registry with custom values
+	echo "Installing mlflow chart from Replicated registry with custom values..."; \
+	helm upgrade --install mlflow-values-test $$OCI_URL/mlflow \
+		--namespace values-test \
+		$$MLFLOW_VALUES_ARGS \
+		--wait --timeout 5m --debug; \
 	\
-	echo "Installing mlflow chart with chart-testing..."; \
-	ct install --config ct-oci.yaml \
-		--charts "mlflow" \
-		--namespace default \
-		--release-label "ci-test"; \
+	# Test the installation
+	echo "Verifying installation..."; \
+	kubectl get pods -n values-test; \
 	\
-	rm -f ct-oci.yaml; \
 	echo "Helm installation with custom values completed successfully."
 
 # Example target to check versions (optional)

applications/mlflow/README.md

@@ -27,8 +27,8 @@ make test
 # Login to Replicated registry
 make registry-login
 
-# Install charts from the Replicated registry (SDK enabled)
-make test-replicated-helm
+# Install charts from the Replicated registry with custom values (SDK enabled)
+make test-replicated-helm-with-values
 ```
 
 ### Environment Variables for Registry Testing
@@ -38,6 +38,9 @@ For testing with charts from the Replicated registry, set:
 - `REPLICATED_CHANNEL` - The channel slug
 - `REPLICATED_LICENSE_ID` - Your Replicated license ID (required for registry authentication)
 
+For the `test-replicated-helm-with-values` target, you can also set:
+- `MLFLOW_VALUES` - Path to a custom values file for MLflow
+
 ### Registry Authentication
 
 The `registry-login` target handles registry authentication using your license ID:
@@ -64,7 +67,7 @@ The pipeline workflow:
 4. `kots-install-test`: Tests KOTS installation
 5. `cleanup-test-release`: Cleans up test resources
 
-> Note: Chart-testing (ct) is set up automatically by the GitHub Action `helm/chart-testing[email protected]` in the CI pipeline. When running locally, ensure you have chart-testing installed.
+The pipeline uses direct Helm commands for installation and testing, ensuring reliable and consistent behavior across environments.
 
 The pipeline is triggered on:
 - Pull requests affecting the MLflow application