wip: feat(e2e): replace Ginkgo e2e tests with Kyverno Chainsaw framework

.github/workflows/test-e2e.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   test-e2e:
-    name: tests-e2e
+    name: Tests E2E
     runs-on: ubuntu-latest
     steps:
       - name: Clone the code
@@ -19,19 +19,49 @@ jobs:
         with:
           go-version-file: go.mod
 
-      - name: Install the latest version of kind
+      - name: Install chainsaw
+        uses: kyverno/action-install-chainsaw@v0.2.12
+
+      - name: Install kind
         run: |
           curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
           chmod +x ./kind
           sudo mv ./kind /usr/local/bin/kind
 
-      - name: Verify kind installation
-        run: kind version
-
       - name: Create kind cluster
-        run: kind create cluster
+        run: kind create cluster --wait 60s
+
+      - name: Build artifact-operator image
+        run: make docker-build OPERATOR=artifact IMG=falcosecurity/artifact-operator:e2e
 
-      - name: Running Test e2e
+      - name: Build falco-operator image
+        run: make docker-build OPERATOR=falco IMG=falcosecurity/falco-operator:e2e ARTIFACT_OPERATOR_IMAGE=falcosecurity/artifact-operator:e2e
+
+      - name: Load images into kind
         run: |
-          go mod tidy
-          make test-e2e
+          kind load docker-image falcosecurity/artifact-operator:e2e
+          kind load docker-image falcosecurity/falco-operator:e2e
+
+      - name: Install CRDs
+        run: make install
+
+      - name: Deploy operator
+        run: make deploy IMG=falcosecurity/falco-operator:e2e
+
+      - name: Wait for operator
+        run: kubectl wait --for=condition=Available deployment/falco-operator -n falco-operator --timeout=120s
+
+      - name: Run e2e tests
+        run: chainsaw test --config test/e2e/chainsaw/.chainsaw.yaml --test-dir test/e2e/chainsaw/
+
+      - name: Collect operator logs
+        if: failure()
+        run: |
+          echo "=== Falco Operator Logs ==="
+          kubectl logs -n falco-operator deployment/falco-operator --tail=200 || true
+          echo "=== Cluster Events ==="
+          kubectl get events --all-namespaces --sort-by='.lastTimestamp' | tail -50 || true
+
+      - name: Cleanup
+        if: always()
+        run: kind delete cluster || true

.gitignore

@@ -25,3 +25,6 @@ go.work
 *.swp
 *.swo
 *~
+
+# Chainsaw test reports
+chainsaw-report.json

Makefile

@@ -79,13 +79,18 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet setup-envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
-# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
-# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
-# Prometheus and CertManager are installed by default; skip with:
-# - PROMETHEUS_INSTALL_SKIP=true
-# - CERT_MANAGER_INSTALL_SKIP=true
+# E2E test configuration using Kind cluster
+E2E_FALCO_IMG ?= falcosecurity/falco-operator:e2e
+E2E_ARTIFACT_IMG ?= falcosecurity/artifact-operator:e2e
+CHAINSAW_TEST_DIR ?= ./test/e2e/chainsaw
+CHAINSAW_CONFIG ?= ./test/e2e/chainsaw/.chainsaw.yaml
+
 .PHONY: test-e2e
-test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
+test-e2e: chainsaw ## Run chainsaw e2e tests (requires running cluster with operator deployed).
+	$(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR)
+
+.PHONY: test-e2e-setup
+test-e2e-setup: manifests generate fmt vet ## Build images and deploy operator to Kind cluster for e2e testing.
 	@command -v kind >/dev/null 2>&1 || { \
 		echo "Kind is not installed. Please install Kind manually."; \
 		exit 1; \
@@ -94,7 +99,26 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated
 		echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
 		exit 1; \
 	}
-	go test ./test/e2e/ -v -ginkgo.v
+	@echo "=== Building artifact-operator image ==="
+	@$(MAKE) docker-build OPERATOR=artifact IMG=$(E2E_ARTIFACT_IMG)
+	@echo "=== Building falco-operator image ==="
+	@$(MAKE) docker-build OPERATOR=falco IMG=$(E2E_FALCO_IMG) ARTIFACT_OPERATOR_IMAGE=$(E2E_ARTIFACT_IMG)
+	@echo "=== Loading images into Kind ==="
+	@kind load docker-image $(E2E_ARTIFACT_IMG)
+	@kind load docker-image $(E2E_FALCO_IMG)
+	@echo "=== Installing CRDs ==="
+	@$(MAKE) install
+	@echo "=== Deploying operator ==="
+	@$(MAKE) deploy IMG=$(E2E_FALCO_IMG)
+	@echo "=== Waiting for operator to be ready ==="
+	@kubectl wait --for=condition=Available deployment/falco-operator -n falco-operator --timeout=120s
+
+.PHONY: test-e2e-teardown
+test-e2e-teardown: ## Undeploy operator after e2e testing.
+	@$(MAKE) undeploy ignore-not-found=true || true
+
+.PHONY: test-e2e-all
+test-e2e-all: test-e2e-setup test-e2e test-e2e-teardown ## Full e2e test lifecycle: setup, test, teardown.
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
@@ -210,6 +234,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
 GCI ?= $(LOCALBIN)/gci
 ADD_LICENSE ?= $(LOCALBIN)/addlicense
+CHAINSAW ?= $(LOCALBIN)/chainsaw
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.5.0
@@ -221,6 +246,7 @@ ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -
 GOLANGCI_LINT_VERSION ?= v2.8.0
 GCI_VERSION ?= v0.13.5
 ADD_LICENSE_VERSION ?= v1.1.1
+CHAINSAW_VERSION ?= v0.2.12
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -259,6 +285,11 @@ $(GCI): $(LOCALBIN)
 addlicense: $(ADD_LICENSE) ## Download addlicense locally if necessary
 $(ADD_LICENSE): $(LOCALBIN)
 	$(call go-install-tool,$(ADD_LICENSE),github.com/google/addlicense,$(ADD_LICENSE_VERSION))
+
+.PHONY: chainsaw
+chainsaw: $(CHAINSAW) ## Download chainsaw locally if necessary.
+$(CHAINSAW): $(LOCALBIN)
+	$(call go-install-tool,$(CHAINSAW),github.com/kyverno/chainsaw,$(CHAINSAW_VERSION))
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed

config/manager/kustomization.yaml

@@ -3,4 +3,6 @@ resources:
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
-
+- name: falcosecurity/falco-operator
+  newName: falcosecurity/falco-operator
+  newTag: e2e

controllers/falco/defaults.go

@@ -74,7 +74,7 @@ var (
 
 	// DefaultFalcoLivenessProbe is the default liveness probe for the Falco container.
 	DefaultFalcoLivenessProbe = &corev1.Probe{
-		InitialDelaySeconds: 60,
+		InitialDelaySeconds: 30,
 		TimeoutSeconds:      5,
 		PeriodSeconds:       15,
 		FailureThreshold:    3,
@@ -89,7 +89,7 @@ var (
 
 	// DefaultFalcoReadinessProbe is the default readiness probe for the Falco container.
 	DefaultFalcoReadinessProbe = &corev1.Probe{
-		InitialDelaySeconds: 30,
+		InitialDelaySeconds: 15,
 		TimeoutSeconds:      5,
 		PeriodSeconds:       15,
 		FailureThreshold:    3,

test/e2e/chainsaw/.chainsaw.yaml

@@ -0,0 +1,36 @@
+# Global Chainsaw configuration file.
+# This file configures default timeouts and settings for all chainsaw e2e tests.
+# Individual tests can override these settings if needed.
+#
+# Note: Some settings like --quiet are CLI-only and configured in the Makefile.
+
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  name: falco-operator-e2e
+spec:
+  # Default timeouts for all operations.
+  # These are generous defaults to account for:
+  # - Slow CI environments
+  # - Falco pod startup time
+  # - OCI artifact download delays
+  timeouts:
+    apply: 30s
+    assert: 5m
+    cleanup: 2m
+    delete: 30s
+    error: 30s
+    exec: 3m
+
+  # Test execution settings.
+  # Can be overridden via CLI: chainsaw test --parallel N
+  parallel: 5
+  failFast: false
+  fullName: true
+  skipDelete: false
+
+  # Debug collection on failure.
+  catch:
+    - events: {}
+    - podLogs:
+        tail: 100