feat(ci): Add e2e test on K3d

.github/workflows/e2e-deploy-k3d.yaml

@@ -0,0 +1,27 @@
+name: E2E Test with on K3d Cluster
+on:
+  - pull_request
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k3d-version: ["v5.6.0", "v5.7.0", "v5.8.1"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup E2E Tests
+        uses: ./.github/workflows/template-e2e-test-k3d
+        with:
+          k3d-version: ${{ matrix.k3d-version }}
+
+      - name: Run E2E Tests
+        run: |
+          make test-e2e

.github/workflows/e2e-deploy.yaml → .github/workflows/e2e-deploy-kind.yaml

@@ -1,4 +1,4 @@
-name: E2E Test with Deploying Tenant and Warehouse
+name: E2E Test with on Kind Cluster
 on:
   - pull_request
 
@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup E2E Tests
-        uses: ./.github/workflows/template-e2e-test
+        uses: ./.github/workflows/template-e2e-test-kind
         with:
           kubernetes-version: ${{ matrix.kubernetes-version }}
 

.github/workflows/template-e2e-test-k3d/action.yaml

@@ -0,0 +1,47 @@
+name: Setup E2E test template
+description: A composite action to setup e2e tests
+
+inputs:
+  k3d-version:
+    required: true
+    description: K3d version
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+
+    - name: Install K3d
+      shell: bash
+      run: |
+        curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=${{ inputs.k3d-version }} bash
+
+    - name: Install Helm
+      uses: azure/setup-helm@v4
+      with:
+        version: v3.14.0
+
+    - name: Install kubectl
+      uses: azure/setup-kubectl@v3
+      with:
+        version: 'latest'
+
+    - name: Build Databend Operator
+      shell: bash
+      run: |
+        ./scripts/gha/build-image.sh
+      env:
+        OPERATOR_CI_IMAGE: datafuselabs/databend-operator:test
+
+    - name: Setup K3d Cluster
+      shell: bash
+      run: |
+        ./scripts/gha/setup-k3d-cluster.sh
+      env:
+        K3D_CLUSTER: databend-operator-cluster
+        OPERATOR_CI_IMAGE: datafuselabs/databend-operator:test
+        REGISTRY_NAME: registry.localhost
+        REGISTRY_PORT: 5000

.github/workflows/template-e2e-test/action.yaml → .github/workflows/template-e2e-test-kind/action.yaml

@@ -14,7 +14,7 @@ runs:
       with:
         go-version-file: go.mod
 
-    - name: Create k8s Kind Cluster
+    - name: Create Kind Cluster
       uses: helm/[email protected]
       with:
         node_image: kindest/node:${{ inputs.kubernetes-version }}
@@ -33,10 +33,10 @@ runs:
       with:
         version: v3.14.0
 
-    - name: Setup Cluster
+    - name: Setup Kind Cluster
       shell: bash
       run: |
-        ./scripts/gha/setup-cluster.sh
+        ./scripts/gha/setup-kind-cluster.sh
       env:
         KIND_CLUSTER: databend-operator-cluster
         OPERATOR_CI_IMAGE: datafuselabs/databend-operator:test

scripts/gha/k3d.yaml

@@ -0,0 +1,26 @@
+apiVersion: k3d.io/v1alpha4
+kind: Simple
+metadata:
+  name: single
+servers: 1
+image: rancher/k3s:v1.29.5-k3s1
+ports:
+  - port: 8888:80
+    nodeFilters:
+      - loadbalancer
+registries:
+  use:
+    - k3d-registry.localhost:5000
+  config: |
+    mirrors:
+      "docker.io":
+        endpoint:
+          - http://host.k3d.internal:5000
+options:
+  k3d:
+    wait: true
+    timeout: "60s"
+    disableLoadbalancer: false
+  kubeconfig:
+    updateDefaultKubeconfig: true
+    switchCurrentContext: true

scripts/gha/setup-k3d-cluster.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Configure variables.
+NAMESPACE="databend-system"
+TIMEOUT="5m"
+REGISTRY_HUB=k3d-${REGISTRY_NAME}:${REGISTRY_PORT}
+
+# Create a k3d cluster.
+echo "Creating k3d registry and cluster"
+k3d registry create ${REGISTRY_NAME} --port 0.0.0.0:${REGISTRY_PORT} -i registry:latest
+k3d cluster create --config ./scripts/gha/k3d.yaml ${K3D_CLUSTER}
+kubectl config use-context k3d-${K3D_CLUSTER}
+
+# Push the databend-operator image to the k3d registry.
+echo "Pushing databend-operator image to k3d registry"
+echo -n "password" | docker login ${REGISTRY_HUB} --username admin --password-stdin
+docker tag ${OPERATOR_CI_IMAGE} ${REGISTRY_HUB}/${OPERATOR_CI_IMAGE}
+docker push ${REGISTRY_HUB}/${OPERATOR_CI_IMAGE}
+
+echo "Update databend operator manifest with newly built image"
+cd manifests
+kustomize edit set image datafuselabs/databend-operator=${REGISTRY_HUB}/${OPERATOR_CI_IMAGE}
+
+echo "Installing databend operator manifests"
+kustomize build . | kubectl apply -f -
+
+(kubectl wait deployment/databend-operator --for=condition=available  -n ${NAMESPACE} --timeout ${TIMEOUT} &&
+  kubectl wait pods --for=condition=ready -n ${NAMESPACE} --timeout ${TIMEOUT} --all) ||
+  (
+    echo "Failed to wait until databend operator is ready" &&
+      kubectl get pods -n ${NAMESPACE} &&
+      kubectl describe pods -n ${NAMESPACE} &&
+      exit 1
+  )
+
+print_cluster_info() {
+  kubectl version
+  kubectl cluster-info
+  kubectl get nodes
+  kubectl get pods -n ${NAMESPACE}
+  kubectl describe pod -n ${NAMESPACE}
+}
+
+print_cluster_info
+
+echo "Deploying meta cluster with helm"
+helm repo add databend https://charts.databend.com
+helm install meta databend/databend-meta --namespace ${NAMESPACE}
+
+(kubectl wait pods --for=condition=ready -n ${NAMESPACE} --timeout ${TIMEOUT} --all) ||
+  (
+    echo "Failed to wait until meta cluster is ready" &&
+      kubectl get pods -n ${NAMESPACE} &&
+      kubectl describe pods -n ${NAMESPACE} &&
+      exit 1
+  )
+
+print_cluster_info