Use k3d to replace kind for faster CI and less disk usage

Signed-off-by: Tao He <[email protected]>

Author: sighingnow

.github/workflows/vineyard-operator.yaml

@@ -100,6 +100,9 @@ jobs:
               exit -1
           fi
 
+          # prepare golangci-lint for faster installation
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0
+
           function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; }
 
           ec make -C k8s golint
@@ -122,7 +125,7 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      
+
       - name: Build docker image
         run: |
           make -C k8s docker-build
@@ -144,6 +147,12 @@ jobs:
           docker tag ${IMG} ${LATEST_IMG}
           docker push ${LATEST_IMG}
 
+      - name: Install k3d
+        if: ${{ matrix.job != 'release' }}
+        run: |
+          curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+          k3d cluster create -c k8s/hack/k3d.yaml
+
       - name: Generate the python image for tests
         if: ${{ matrix.job != 'release' }}
         run: |
@@ -202,4 +211,9 @@ jobs:
       - name: e2e-tests-workflow
         if: ${{ matrix.job == 'e2e-tests-workflow' }}
         run: |
-          make -C k8s e2e-tests-workflow
+          make -C k8s e2e-tests-workflow
+
+      - name: Uninstall k3d
+        if: ${{ always() }}
+        run: |
+          k3d cluster delete -c k8s/hack/k3d.yaml || true

k8s/Makefile

@@ -54,20 +54,6 @@ all: manager
 check: lint e2e-test
 .PHONY: check
 
-# Build a 4-nodes(1 master and 3 workers) kind cluster with local registry
-build-local-cluster:
-	@touch /tmp/e2e-k8s.config
-	@echo "the kubeconfig path is /tmp/e2e-k8s.config"
-	@echo "Creating the kind cluster with local registry"
-	@bash test/hack/build-kind-cluster-with-local-registry.sh
-.PHONY: build-local-cluster
-
-delete-local-cluster:
-	@rm /tmp/e2e-k8s.config
-	@echo "Deleting the kind cluster with local registry"
-	@bash test/hack/delete-kind-cluster-with-local-registry.sh
-.PHONY: delete-local-cluster
-
 prepare-e2e-test: e2e
 	@echo "Install e2e test dependencies..."
 	@sudo bash test/hack/prepare-e2e.sh
@@ -78,118 +64,104 @@ prepare-e2e-test: e2e
 # install the cert-manager and wait for ready
 install-cert-manager:
 	@echo "Installing cert-manager..."
-	@kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml --kubeconfig=/tmp/e2e-k8s.config
-	@kubectl wait pods -n cert-manager --all --for condition=Ready --timeout=300s --kubeconfig=/tmp/e2e-k8s.config
+	@kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml
+	@kubectl wait pods -n cert-manager --all --for condition=Ready --timeout=300s
 	@echo "Cert-Manager ready."
 
 # install the vineyard-operator and wait for ready
 install-vineyard-operator:
 	@echo "Installing vineyard-operator..."
-	@docker tag vineyardcloudnative/vineyard-operator:latest localhost:5001/vineyard-operator:latest
-	@docker push localhost:5001/vineyard-operator:latest
-	@export IMG=localhost:5001/vineyard-operator:latest KUBECONFIG=/tmp/e2e-k8s.config && make deploy
-	@kubectl wait deployment/vineyard-controller-manager --for condition=Available -n vineyard-system --timeout=300s --kubeconfig=/tmp/e2e-k8s.config
+	@make deploy
+	@kubectl wait deployment/vineyard-controller-manager --for condition=Available -n vineyard-system --timeout=300s
 	@echo "Vineyard-Operator Ready"
 
 # install the vineyard cluster and wait for ready
 install-vineyard-cluster:
 	@echo "Installing vineyard cluster..."
-	@docker tag vineyardcloudnative/vineyardd:alpine-latest localhost:5001/vineyardd:alpine-latest
-	@docker push localhost:5001/vineyardd:alpine-latest
-	@kubectl apply -f test/e2e/vineyardd.yaml --kubeconfig=/tmp/e2e-k8s.config
-	@kubectl wait vineyardd/vineyardd-sample --for condition=Available -n vineyard-system --timeout=300s --kubeconfig=/tmp/e2e-k8s.config
+	@kubectl apply -f test/e2e/vineyardd.yaml
+	@kubectl wait vineyardd/vineyardd-sample --for condition=Available -n vineyard-system --timeout=300s
 	@echo "Vineyard cluster Ready"
 
-install-vineyard: build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+install-vineyard: install-cert-manager install-vineyard-operator install-vineyard-cluster
 	sleep 6000
-e2e-tests-assembly-local: kustomize prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-assembly-local: kustomize prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running local assembly e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/local-assembly-e2e.yaml
 	@echo "local assembly e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-assembly-local
 
-e2e-tests-assembly-distributed: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-assembly-distributed: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running distributed assembly e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/distributed-assembly-e2e.yaml
 	@echo "distributed assembly e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-assembly-distributed
 
-e2e-tests-autogenerated-helm-chart: prepare-e2e-test build-local-cluster install-cert-manager
+e2e-tests-autogenerated-helm-chart: prepare-e2e-test install-cert-manager
 	@echo "Running autogenerated helm chart e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/autogenerated-helm-chart/e2e.yaml
 	@echo "autogenerated helm chart e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-autogenerated-helm-chart
 
-e2e-tests-failover: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-failover: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running failover e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/failover/e2e.yaml
 	@echo "failover e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-failover
 
-e2e-tests-repartition-dask: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-repartition-dask: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running repartition dask e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/repartition/dask-repartition-e2e.yaml
 	@echo "repartition dask e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-repartition-dask
 
-e2e-tests-scheduler-outside-cluster: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-scheduler-outside-cluster: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running scheduler outside cluster e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/scheduling-outside-cluster/e2e.yaml
 	@echo "scheduler outside cluster e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-scheduler-outside-cluster
 
-e2e-tests-serialize: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-serialize: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running serialize e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/serialize/e2e.yaml
 	@echo "serialize e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-serialize
 
-e2e-tests-sidecar: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator
+e2e-tests-sidecar: prepare-e2e-test install-cert-manager install-vineyard-operator
 	@echo "Running sidecar e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/sidecar/e2e.yaml
 	@echo "sidecar e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-sidecar
 
-e2e-tests-spill: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator
+e2e-tests-spill: prepare-e2e-test install-cert-manager install-vineyard-operator
 	@echo "Running spill e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/spill/e2e.yaml
 	@echo "spill e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-spill
 
-e2e-tests-workflow: prepare-e2e-test build-local-cluster install-cert-manager install-vineyard-operator install-vineyard-cluster
+e2e-tests-workflow: prepare-e2e-test install-cert-manager install-vineyard-operator install-vineyard-cluster
 	@echo "Running workflow e2e test..."
 	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/workflow/e2e.yaml
 	@echo "workflow e2e test passed."
 	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
 	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	@make delete-local-cluster
 .PHONY: e2e-tests-workflow
 
 # Build manager binary

k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,30 @@
+apiVersion: k3d.io/v1alpha3
+kind: Simple
+name: cluster
+servers: 1
+agents: 3
+image: rancher/k3s:latest
+volumes: # repeatable flags are represented as YAML lists
+  - volume: /var/run/docker.sock:/var/run/docker.sock
+    nodeFilters:
+      - server:*
+      - agent:*
+options:
+  k3d:
+    wait: true 
+    timeout: "600s"
+    disableLoadbalancer: false
+    disableImageVolume: false
+    disableRollback: false
+    loadbalancer:
+      configOverrides:
+        - settings.workerConnections=2048
+  k3s:
+    extraArgs:
+      - arg: --docker
+        nodeFilters:
+          - server:*
+          - agent:*
+  kubeconfig:
+    updateDefaultKubeconfig: true
+    switchCurrentContext: true

k8s/hack/k3d.yaml

@@ -15,7 +15,6 @@ push-%: %
 
 local-assembly-images: local-assembly assembly-job1 assembly-job2
 publish-local-assembly-images: push-local-assembly push-assembly-job1 push-assembly-job2
-e2e-test-assembly: push-local-assembly push-assembly-job1 push-assembly-job2
 
 local-assembly:
 	docker build assembly-demo/ -f Dockerfile \
@@ -39,7 +38,6 @@ assembly-job2:
 
 distributed-assembly-images: distributed-assembly distributed-assembly-job1 distributed-assembly-job2 distributed-assembly-job3
 publish-distributed-assembly-images: push-distributed-assembly push-distributed-assembly-job1 push-distributed-assembly-job2 push-distributed-assembly-job3
-e2e-test-distributed-assembly: push-local-assembly push-distributed-assembly push-distributed-assembly-job1 push-distributed-assembly-job2 push-distributed-assembly-job3
 
 distributed-assembly:
 	docker build assembly-demo/ -f Dockerfile \
@@ -71,7 +69,6 @@ distributed-assembly-job3:
 
 serialize-images: serialize-job
 publish-serialize-images: push-serialize-job
-e2e-test-serialize: push-serialize-job
 
 serialize-job:
 	docker build . -f Dockerfile \
@@ -83,7 +80,6 @@ serialize-job:
 
 failover-images: backup-job recover-job build-local-object build-distributed-object-step1 build-distributed-object-step2 get-local-object get-distributed-object
 publish-failover-images: push-backup-job push-recover-job push-build-local-object push-build-distributed-object-step1 push-build-distributed-object-step2 push-get-local-object push-get-distributed-object
-e2e-test-failover: push-backup-job push-recover-job push-build-local-object push-build-distributed-object-step1 push-build-distributed-object-step2 push-get-local-object push-get-distributed-object
 
 backup-job:
 	docker build failover-demo/ -f Dockerfile \
@@ -131,7 +127,6 @@ get-distributed-object:
 
 repartition-images: dask-repartition dask-repartition-job1 dask-repartition-job2 dask-worker-with-vineyard
 publish-repartition-images: push-dask-repartition push-dask-repartition-job1 push-dask-repartition-job2 push-dask-worker-with-vineyard
-e2e-test-repartition: push-dask-repartition push-dask-repartition-job1 push-dask-repartition-job2 push-dask-worker-with-vineyard
 
 dask-repartition:
 	docker build repartition-demo/ -f Dockerfile \
@@ -163,7 +158,6 @@ dask-worker-with-vineyard:
 
 sidecar-images: sidecar-job
 publish-sidecar-images: push-sidecar-job
-e2e-test-sidecar: push-sidecar-job
 
 sidecar-job:
 	docker build sidecar-demo/ -f Dockerfile \
@@ -175,7 +169,6 @@ sidecar-job:
 
 workflow-images: workflow-job1 workflow-job2
 publish-workflow-images: push-workflow-job1 push-workflow-job2
-e2e-test-workflow: push-workflow-job1 push-workflow-job2
 
 workflow-job1:
 	docker build workflow-demo/ -f Dockerfile \
@@ -189,15 +182,13 @@ workflow-job2:
 		-t $(REGISTRY)/workflow-job2:$(TAG)
 .PHONY: workflow-job2
 
-# publish all images to ghcr registry
-ALL-IMAGES ?= \
-publish-local-assembly-images \
-publish-distributed-assembly-images \
-publish-serialize-images \
-publish-failover-images \
-publish-repartition-images \
-publish-sidecar-images \
-publish-workflow-images
-
-publish-all-images: $(ALL-IMAGES)
-.PHONY: publish-all-images
+############# publish all images to ghcr registry #############################
+
+publish-all-images: publish-local-assembly-images \
+					publish-distributed-assembly-images \
+					publish-serialize-images \
+					publish-failover-images \
+					publish-repartition-images \
+					publish-sidecar-images \
+					publish-workflow-images
+.PHONY: publish-all-images

k8s/test/e2e/Makefile

@@ -44,7 +44,7 @@ distributedJob1:
     jobName: none
     dataName: ""
   schedulerName: ""
-  nodeName: kind-worker
+  nodeName: k3d-cluster-agent-0
 
 
 distributedJob2:
@@ -56,7 +56,7 @@ distributedJob2:
     jobName: none
     dataName: ""
   schedulerName: ""
-  nodeName: kind-worker2
+  nodeName: k3d-cluster-agent-1
 
 distributedJob3:
   jobName: distributed-assembly-job3

k8s/test/e2e/assembly-demo/assembly-job-config.yaml

@@ -49,7 +49,7 @@ spec:
       {{- end }}
       containers:
         - name: {{ (datasource "config").$job.jobName }}
-          image: localhost:5001/{{ (datasource "config").$job.jobName }}
+          image: ghcr.io/v6d-io/v6d/{{ (datasource "config").$job.jobName }}
           env:
             - name: JOB_NAME
               value: {{ (datasource "config").$job.jobName }}

k8s/test/e2e/assembly-demo/assembly-job.yaml

@@ -16,11 +16,11 @@
 # Test assembly operation in different node
 setup:
   env: kind
-  kubeconfig: /tmp/e2e-k8s.config
+  kubeconfig: ~/.kube/config
   steps:
     - name: download all distributed assembly images into kind cluster
       command: |
-        make -C k8s/test/e2e e2e-test-distributed-assembly REGISTRY=localhost:5001
+        make -C k8s/test/e2e distributed-assembly-images
     - name: install job1 and job2
       command: |
         kubectl create namespace vineyard-job

k8s/test/e2e/assembly/distributed-assembly-e2e.yaml

@@ -16,11 +16,11 @@
 # Test assembly operation in the same node
 setup:
   env: kind
-  kubeconfig: /tmp/e2e-k8s.config
+  kubeconfig: ~/.kube/config
   steps:
     - name: download all local assembly images into kind cluster
       command: |
-        make -C k8s/test/e2e e2e-test-assembly REGISTRY=localhost:5001
+        make -C k8s/test/e2e local-assembly-images
     - name: install job1
       command: |
         kubectl create namespace vineyard-job