Skip to content

Commit bae93c1

Browse files
ciarams87ciarams87
authored
Run conformance tests in pod on cluster (#787)
* Run conformance tests in pod on cluster
1 parent e04cc27 commit bae93c1

File tree

5 files changed

+153
-44
lines changed

5 files changed

+153
-44
lines changed

‎conformance/Makefile

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ EXEMPT_FEATURES = ReferenceGrant
66
KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind
77
TAG = latest
88
PREFIX = conformance-test-runner
9+
NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml
10+
NGINX_IMAGE=$(shell yq '.spec.template.spec.containers[1].image as $$nginx_ver | $$nginx_ver' $(NKG_DEPLOYMENT_MANIFEST))
911
.DEFAULT_GOAL := help
1012

1113
.PHONY: help
@@ -21,13 +23,19 @@ create-kind-cluster: ## Create a kind cluster
2123
kind create cluster --image kindest/node:v1.27.1
2224
kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
2325

24-
.PHONY: prepare-nkg
25-
prepare-nkg: ## Build and load NKG container on configured kind cluster
26+
.PHONY: preload-nginx-container
27+
preload-nginx-container: ## Preload NGINX container on configured kind cluster
28+
docker pull $(NGINX_IMAGE)
29+
kind load docker-image $(NGINX_IMAGE)
30+
31+
.PHONY: build-and-load-images
32+
build-and-load-images: preload-nginx-container ## Build NKG container and load it and NGINX container on configured kind cluster
33+
yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST)
2634
cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container
2735
kind load docker-image $(NKG_PREFIX):$(NKG_TAG)
2836

29-
.PHONY: install-nkg
30-
install-nkg: ## Install NKG with provisioner on configured kind cluster
37+
.PHONY: prepare-nkg-dependencies
38+
prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster
3139
kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
3240
kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system
3341
kubectl apply -f ../deploy/manifests/namespace.yaml
@@ -36,16 +44,30 @@ install-nkg: ## Install NKG with provisioner on configured kind cluster
3644
kubectl apply -f ../deploy/manifests/rbac.yaml
3745
kubectl apply -f ../deploy/manifests/gatewayclass.yaml
3846
kubectl apply -f ../deploy/manifests/service/nodeport.yaml
39-
kubectl apply -f provisioner/provisioner.yaml
4047

41-
.PHONY: update-test-kind-config
42-
update-test-kind-config: ## Update kind config
43-
sed -ir "s|server:.*|server: https://kind-control-plane:6443|" $(KIND_KUBE_CONFIG_FOLDER)/config
48+
.PHONY: install-nkg-local-build
49+
install-nkg-local-build: build-and-load-images prepare-nkg-dependencies ## Install NKG from local build with provisioner on configured kind cluster
50+
yq '(select(di != 3))' provisioner/provisioner.yaml | kubectl apply -f -
51+
yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' provisioner/provisioner.yaml | kubectl apply -f -
52+
53+
.PHONY: install-nkg-edge
54+
install-nkg-edge: preload-nginx-container prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster
55+
kubectl apply -f provisioner/provisioner.yaml
4456

4557
.PHONY: run-conformance-tests
46-
run-conformance-tests: update-test-kind-config ## Run conformance tests
47-
docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) \
48-
go test -timeout 25m -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug --supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES)
58+
run-conformance-tests: ## Run conformance tests
59+
kind load docker-image $(PREFIX):$(TAG)
60+
kubectl apply -f tests/conformance-rbac.yaml
61+
kubectl run -i conformance \
62+
--image=$(PREFIX):$(TAG) --image-pull-policy=Never \
63+
--overrides='{ "spec": { "serviceAccountName": "conformance" } }' \
64+
--restart=Never -- go test -v . -tags conformance -args --gateway-class=$(GATEWAY_CLASS) --debug \
65+
--supported-features=$(SUPPORTED_FEATURES) --exempt-features=$(EXEMPT_FEATURES)
66+
67+
.PHONY: cleanup-conformance-tests
68+
cleanup-conformance-tests: ## Clean up conformance tests fixtures
69+
kubectl delete pod conformance
70+
kubectl delete -f tests/conformance-rbac.yaml
4971

5072
.PHONY: uninstall-nkg
5173
uninstall-nkg: ## Uninstall NKG on configured kind cluster
@@ -54,7 +76,11 @@ uninstall-nkg: ## Uninstall NKG on configured kind cluster
5476
kubectl delete -f ../deploy/manifests/namespace.yaml
5577
kubectl delete clusterrole nginx-gateway-provisioner
5678
kubectl delete clusterrolebinding nginx-gateway-provisioner
57-
79+
80+
.PHONY: undo-image-update
81+
undo-image-update: ## Undo the NKG image name and tag in deployment manifest
82+
git checkout -- $(NKG_DEPLOYMENT_MANIFEST)
83+
5884
.PHONY: delete-kind-cluster
5985
delete-kind-cluster: ## Delete kind cluster
6086
kind delete cluster

‎conformance/README.md

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* [kind](https://kind.sigs.k8s.io/).
66
* Docker.
77
* Golang.
8+
* [yq](https://github.com/mikefarah/yq/#install)
89

910
**Note**: all commands in steps below are executed from the ```conformance``` directory
1011

@@ -13,66 +14,80 @@ List available commands:
1314
```bash
1415
$ make
1516

17+
build-and-load-images Build NKG container and load it and NGINX container on configured kind cluster
1618
build-test-runner-image Build conformance test runner image
19+
cleanup-conformance-tests Clean up conformance tests fixtures
1720
create-kind-cluster Create a kind cluster
1821
delete-kind-cluster Delete kind cluster
1922
help Display this help
20-
install-nkg Install NKG with provisioner on configured kind cluster
21-
prepare-nkg Build and load NKG container on configured kind cluster
23+
install-nkg-edge Install NKG with provisioner from edge on configured kind cluster
24+
install-nkg-local-build Install NKG from local build with provisioner on configured kind cluster
25+
preload-nginx-container Preload NGINX container on configured kind cluster
26+
prepare-nkg-dependencies Install NKG dependencies on configured kind cluster
2227
run-conformance-tests Run conformance tests
28+
undo-image-update Undo the NKG image name and tag in deployment manifest
2329
uninstall-nkg Uninstall NKG on configured kind cluster
24-
update-test-kind-config Update kind config
2530
```
31+
32+
**Note:** The following variables are configurable when running the below `make` commands:
33+
34+
| Variable | Default | Description |
35+
| ------------- | ------------- | ------------- |
36+
| TAG | latest | The tag for the conformance test image |
37+
| PREFIX | conformance-test-runner | The prefix for the conformance test image |
38+
| NKG_TAG | edge | The tag for the locally built NKG image |
39+
| NKG_PREFIX | nginx-kubernetes-gateway | The prefix for the locally built NKG image |
40+
| KIND_KUBE_CONFIG_FOLDER | ~/.kube/kind | The location of the kubeconfig folder |
41+
| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests |
42+
| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. |
43+
| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests |
44+
| NGINX_IMAGE | as defined in the ../deploy/manifests/deployment.yaml file | The NGINX image for the NKG deployments |
45+
| NKG_DEPLOYMENT_MANIFEST | ../deploy/manifests/deployment.yaml | The location of the NKG deployment manifest |
46+
2647
### Step 1 - Create a kind Cluster
2748

2849
```bash
2950
$ make create-kind-cluster
3051
```
52+
### Step 2 - Install Nginx Kubernetes Gateway to configured kind cluster
3153

32-
### Step 2 - Update NKG deployment and provisioner manifests
33-
**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway
34-
* Set NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> to preferred values.
35-
* Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block.
36-
* Navigate to `conformance/provisioner` and update values in `provisioner.yaml` as specified in below code-block.
37-
* Save the changes.
38-
```
39-
.
40-
..
41-
containers:
42-
- image: <repo_name>:<image_tag>
43-
imagePullPolicy: Never
44-
..
45-
.
54+
#### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster
55+
```bash
56+
$ make install-nkg-local-build
4657
```
4758

48-
### Step 3 - Build and load Nginx Kubernetes Gateway container to configured kind cluster
49-
**Note**: this step is only required when user wants to run conformance tests using locally built image of Nginx Kubernetes Gateway
59+
#### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster
60+
Instead of the above command, you can skip the build NKG image step and prepare the environment to instead
61+
use the `edge` image
5062

5163
```bash
52-
$ make NKG_PREFIX=<repo_name> NKG_TAG=<image_tag> prepare-nkg
53-
64+
$ make install-nkg-edge
5465
```
55-
### Step 4 - Build conformance test runner image
66+
67+
### Step 3 - Build conformance test runner image
5668
```bash
5769
$ make build-test-runner-image
5870
```
5971

60-
### Step 5 - Install Nginx Kubernetes Gateway
72+
### Step 4 - Run Gateway conformance tests
6173
```bash
62-
$ make install-nkg
74+
$ make run-conformance-tests
6375
```
6476

65-
### Step 6 - Run Gateway conformance tests
77+
### Step 5 - Cleanup the conformance test fixtures and uninstall Nginx Kubernetes Gateway
6678
```bash
67-
$ make run-conformance-tests
79+
$ make cleanup-conformance-tests
80+
$ make uninstall-nkg
6881
```
6982

70-
### Step 7 - Uninstall Nginx Kubernetes Gateway
83+
### Step 6 - Revert changes to the NKG deployment manifest
84+
**Optional** Not required if using `edge` image
85+
**Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file!
7186
```bash
72-
$ make uninstall-nkg
87+
$ make undo-image-update
7388
```
7489

75-
### Step 8 - Delete kind cluster
90+
### Step 7 - Delete kind cluster
7691
```bash
7792
$ make delete-kind-cluster
7893
```

‎conformance/tests/conformance-rbac.yaml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: conformance
5+
---
6+
kind: ClusterRole
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
metadata:
9+
name: conformance
10+
rules:
11+
- apiGroups:
12+
- ""
13+
resources:
14+
- namespaces
15+
- pods
16+
- secrets
17+
- services
18+
verbs:
19+
- create
20+
- delete
21+
- get
22+
- list
23+
- update
24+
- apiGroups:
25+
- apps
26+
resources:
27+
- deployments
28+
verbs:
29+
- create
30+
- delete
31+
- get
32+
- list
33+
- apiGroups:
34+
- gateway.networking.k8s.io
35+
resources:
36+
- gatewayclasses
37+
verbs:
38+
- get
39+
- list
40+
- apiGroups:
41+
- gateway.networking.k8s.io
42+
resources:
43+
- gateways
44+
- httproutes
45+
verbs:
46+
- create
47+
- delete
48+
- get
49+
- list
50+
- patch
51+
---
52+
kind: ClusterRoleBinding
53+
apiVersion: rbac.authorization.k8s.io/v1
54+
metadata:
55+
name: nginx-conformance
56+
subjects:
57+
- kind: ServiceAccount
58+
name: conformance
59+
namespace: default
60+
roleRef:
61+
kind: ClusterRole
62+
name: conformance
63+
apiGroup: rbac.authorization.k8s.io

‎docs/developer/quickstart.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Follow these steps to set up your development environment.
1414
- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
1515
- [git](https://git-scm.com/)
1616
- [GNU Make](https://www.gnu.org/software/software.html)
17+
- [yq](https://github.com/mikefarah/yq/#install)
1718
- [fieldalignment](https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/fieldalignment):
1819

1920
```shell
@@ -92,6 +93,10 @@ make unit-test
9293

9394
For more details on testing, see the [testing](/docs/developer/testing.md) documentation.
9495

96+
## Gateway API Conformance Testing
97+
98+
To run Gateway API conformance tests, please follow the instructions on [this](/conformance/README.md) page.
99+
95100
## Run the Linter
96101

97102
To lint the code, run the following make command from the project's root directory:

‎docs/developer/testing.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
# Testing
22

33
This document provides guidelines for testing, including instructions on running the unit tests, accessing the code
4-
coverage report, and performing manual testing. By following these guidelines, you will gain a thorough understanding of
5-
the project's approach to unit testing, enabling you to ensure code quality, validate functionality, and maintain robust
6-
test coverage.
4+
coverage report, performing manual testing, and running the conformance tests. By following these guidelines, you will
5+
gain a thorough understanding of the project's approach to unit testing, enabling you to ensure code quality, validate
6+
functionality, and maintain robust test coverage.
77

88
## Unit Test Guidelines
99

0 commit comments

Comments
 (0)