chore: improve e2e tests (#521)

Signed-off-by: Kavindu Dodanduwa <[email protected]>
Signed-off-by: Kavindu Dodanduwa <[email protected]>
Co-authored-by: Florian Bacher <[email protected]>

Author: Kavindu-Dodan

Makefile

@@ -65,20 +65,23 @@ vet: ## Run go vet against code.
 unit-test: manifests fmt vet generate envtest ## Run tests.
 	go test ./... -v -short -coverprofile cover.out
 
-## Requires the operator to be deployed
-.PHONY: e2e-test
-e2e-test: manifests generate fmt vet
-	kubectl -n open-feature-operator-system apply -f ./test/e2e/e2e.yml
-	kubectl wait --for=condition=Available=True deploy --all -n 'open-feature-operator-system'
-	./test/e2e/run.sh
-
-.PHONY: e2e-test-kuttl #these tests should run on a real cluster!
+## e2e tests require the operator to be deployed in a real cluster
+.PHONY: e2e-test-kuttl
 e2e-test-kuttl:
-	kubectl kuttl test --start-kind=false ./test/e2e/kuttl --config=./kuttl-test.yaml
+	kubectl kuttl test --start-kind=false --config=./kuttl-test.yaml
 
-.PHONY: e2e-test-kuttl-local #these tests should run on a real cluster!
+.PHONY: e2e-test-kuttl-local
 e2e-test-kuttl-local:
-	kubectl kuttl test --start-kind=false ./test/e2e/kuttl/scenarios --config=./kuttl-test-local.yaml
+	kubectl kuttl test --start-kind=false --config=./kuttl-test-local.yaml
+
+.PHONY: e2e-test-validate-local
+e2e-test-validate-local:
+	docker build . -t open-feature-operator-local:validate
+	kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests
+	kind load docker-image open-feature-operator-local:validate --name e2e-tests
+	IMG=open-feature-operator-local:validate make deploy-operator
+	IMG=open-feature-operator-local:validate make e2e-test-kuttl
+	kind delete cluster --name e2e-tests
 
 .PHONY: lint
 lint:

docs/development_notes.md

@@ -13,7 +13,7 @@ Run `make test` to run the test suite. The controller integration tests use [env
 This provides means of asserting that the Kubernetes components reach the desired state without the overhead of using an actual cluster, keeping
 test runtime and resource consumption down.
 
-An e2e test suite can also be found in the [`/test/e2e`](../test/e2e/DEVELOPER.md) directory. These tests are run as part of the `pr-lint` github action, they work by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator.
+An e2e test suite can also be found in the [`/test/e2e`](../test/e2e/README.md) directory. These tests are run as part of the `pr-checks` github action, they work by deploying an nginx reverse proxy and asserting that curls to the proxy elicit expected behaviour from the flagd sidecar created by open-feature-operator.
 
 ## Releases
 

kuttl-test-local.yaml

@@ -2,5 +2,5 @@ apiVersion: kuttl.dev/v1
 kind: TestSuite
 crdDir: ./config/crd/bases
 testDirs:
-  - ./test/e2e/kuttl/scenarios/
+  - ./test/e2e/kuttl
 timeout: 300

kuttl-test.yaml

@@ -2,6 +2,6 @@ apiVersion: kuttl.dev/v1
 kind: TestSuite
 crdDir: ./config/crd/bases
 testDirs:
-  - ./test/e2e/kuttl/scenarios/
+  - ./test/e2e/kuttl
 timeout: 300
-skipDelete: true
+skipDelete: true # skip removing test related resources (ex:- namespaces)

test/e2e/DEVELOPER.md

@@ -0,0 +1,41 @@
+# E2E Testing
+
+This suite tests the end-to-end operation of the open-feature-operator.
+
+Tests are written with [kuttl](https://kuttl.dev/) and assertions are executed from a curl enabled Job.
+Ngnix reverse proxy is used as the workload where flagd get injected using OFO annotations.
+
+## Running and validating locally
+
+It is recommended to run and validate e2e test locally before opening a pull request.
+
+To run locally (commands are executed from the project root level),
+
+1. Build the operator locally - `docker build . -t open-feature-operator-local:validate`
+2. Create a kind cluster - `kind create cluster --config ./test/e2e/kind-cluster.yml --name e2e-tests`
+3. Load locally build operator image - `kind load docker-image open-feature-operator-local:validate --name e2e-tests`
+4. Deploy Operator to kind cluster - `IMG=open-feature-operator-local:validate make deploy-operator`
+5. Execute kuttl tests - `IMG=open-feature-operator-local:validate make e2e-test-kuttl`
+
+Alternatively, you can use `e2e-test-validate-local` Makefile rule to execute all above and cleanup the kind cluster,
+
+> make e2e-test-validate-local
+
+After the test run, make sure test status by validating kuttl output,
+
+```text
+--- PASS: kuttl (48.71s)
+    --- PASS: kuttl/harness (0.00s)
+        --- PASS: kuttl/harness/assets (0.01s)
+        --- PASS: kuttl/harness/flagd-disabled (12.58s)
+        --- PASS: kuttl/harness/inject-flagd (26.41s)
+        --- PASS: kuttl/harness/fsconfig-file-sync (31.73s)
+        --- PASS: kuttl/harness/fsconfig-k8s-sync (31.74s)
+        --- PASS: kuttl/harness/fsconfig-flagd-proxy-sync (48.49s)
+```
+
+### Running individual tests
+
+You can use kuttl command options to execute individual tests. Consider the example command below,
+
+>$ kubectl kuttl test --start-kind=false ./test/e2e/kuttl --config=kuttl-test.yaml --test=flagd-disabled

test/e2e/README.md

@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE

test/e2e/e2e.yml

@@ -2,3 +2,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - command: kubectl wait --for=condition=complete job flagd-query-test -n $NAMESPACE
+collectors:
+  - command: kubectl logs -l job-name=flagd-query-test -n $NAMESPACE