diff --git a/.github/workflows/e2e-deploy-k3d.yaml b/.github/workflows/e2e-deploy-k3d.yaml new file mode 100644 index 0000000..63ecac2 --- /dev/null +++ b/.github/workflows/e2e-deploy-k3d.yaml @@ -0,0 +1,27 @@ +name: E2E Test with on K3d Cluster +on: + - pull_request + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + k3d-version: ["v5.6.0", "v5.7.0", "v5.8.1"] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup E2E Tests + uses: ./.github/workflows/template-e2e-test-k3d + with: + k3d-version: ${{ matrix.k3d-version }} + + - name: Run E2E Tests + run: | + make test-e2e diff --git a/.github/workflows/e2e-deploy.yaml b/.github/workflows/e2e-deploy-kind.yaml similarity index 83% rename from .github/workflows/e2e-deploy.yaml rename to .github/workflows/e2e-deploy-kind.yaml index 07f63e1..c3d4fb7 100644 --- a/.github/workflows/e2e-deploy.yaml +++ b/.github/workflows/e2e-deploy-kind.yaml @@ -1,4 +1,4 @@ -name: E2E Test with Deploying Tenant and Warehouse +name: E2E Test with on Kind Cluster on: - pull_request @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 - name: Setup E2E Tests - uses: ./.github/workflows/template-e2e-test + uses: ./.github/workflows/template-e2e-test-kind with: kubernetes-version: ${{ matrix.kubernetes-version }} diff --git a/.github/workflows/template-e2e-test-k3d/action.yaml b/.github/workflows/template-e2e-test-k3d/action.yaml new file mode 100644 index 0000000..ffed068 --- /dev/null +++ b/.github/workflows/template-e2e-test-k3d/action.yaml @@ -0,0 +1,47 @@ +name: Setup E2E test template +description: A composite action to setup e2e tests + +inputs: + k3d-version: + required: true + description: K3d version + +runs: + using: composite + steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install K3d + shell: bash + run: | + curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=${{ inputs.k3d-version }} bash + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: v3.14.0 + + - name: Install kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'latest' + + - name: Build Databend Operator + shell: bash + run: | + ./scripts/gha/build-image.sh + env: + OPERATOR_CI_IMAGE: datafuselabs/databend-operator:test + + - name: Setup K3d Cluster + shell: bash + run: | + ./scripts/gha/setup-k3d-cluster.sh + env: + K3D_CLUSTER: databend-operator-cluster + OPERATOR_CI_IMAGE: datafuselabs/databend-operator:test + REGISTRY_NAME: registry.localhost + REGISTRY_PORT: 5000 diff --git a/.github/workflows/template-e2e-test/action.yaml b/.github/workflows/template-e2e-test-kind/action.yaml similarity index 90% rename from .github/workflows/template-e2e-test/action.yaml rename to .github/workflows/template-e2e-test-kind/action.yaml index a6f19cc..360e425 100644 --- a/.github/workflows/template-e2e-test/action.yaml +++ b/.github/workflows/template-e2e-test-kind/action.yaml @@ -14,7 +14,7 @@ runs: with: go-version-file: go.mod - - name: Create k8s Kind Cluster + - name: Create Kind Cluster uses: helm/kind-action@v1.11.0 with: node_image: kindest/node:${{ inputs.kubernetes-version }} @@ -33,10 +33,10 @@ runs: with: version: v3.14.0 - - name: Setup Cluster + - name: Setup Kind Cluster shell: bash run: | - ./scripts/gha/setup-cluster.sh + ./scripts/gha/setup-kind-cluster.sh env: KIND_CLUSTER: databend-operator-cluster OPERATOR_CI_IMAGE: datafuselabs/databend-operator:test diff --git a/scripts/gha/k3d.yaml b/scripts/gha/k3d.yaml new file mode 100644 index 0000000..53a795b --- /dev/null +++ b/scripts/gha/k3d.yaml @@ -0,0 +1,26 @@ +apiVersion: k3d.io/v1alpha4 +kind: Simple +metadata: + name: single +servers: 1 +image: rancher/k3s:v1.29.5-k3s1 +ports: + - port: 8888:80 + nodeFilters: + - loadbalancer +registries: + use: + - k3d-registry.localhost:5000 + config: | + mirrors: + "docker.io": + endpoint: + - http://host.k3d.internal:5000 +options: + k3d: + wait: true + timeout: "60s" + disableLoadbalancer: false + kubeconfig: + updateDefaultKubeconfig: true + switchCurrentContext: true diff --git a/scripts/gha/setup-k3d-cluster.sh b/scripts/gha/setup-k3d-cluster.sh new file mode 100755 index 0000000..1cf6ac6 --- /dev/null +++ b/scripts/gha/setup-k3d-cluster.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# Configure variables. +NAMESPACE="databend-system" +TIMEOUT="5m" +REGISTRY_HUB=k3d-${REGISTRY_NAME}:${REGISTRY_PORT} + +# Create a k3d cluster. +echo "Creating k3d registry and cluster" +k3d registry create ${REGISTRY_NAME} --port 0.0.0.0:${REGISTRY_PORT} -i registry:latest +k3d cluster create --config ./scripts/gha/k3d.yaml ${K3D_CLUSTER} +kubectl config use-context k3d-${K3D_CLUSTER} + +# Push the databend-operator image to the k3d registry. +echo "Pushing databend-operator image to k3d registry" +echo -n "password" | docker login ${REGISTRY_HUB} --username admin --password-stdin +docker tag ${OPERATOR_CI_IMAGE} ${REGISTRY_HUB}/${OPERATOR_CI_IMAGE} +docker push ${REGISTRY_HUB}/${OPERATOR_CI_IMAGE} + +echo "Update databend operator manifest with newly built image" +cd manifests +kustomize edit set image datafuselabs/databend-operator=${REGISTRY_HUB}/${OPERATOR_CI_IMAGE} + +echo "Installing databend operator manifests" +kustomize build . | kubectl apply -f - + +(kubectl wait deployment/databend-operator --for=condition=available -n ${NAMESPACE} --timeout ${TIMEOUT} && + kubectl wait pods --for=condition=ready -n ${NAMESPACE} --timeout ${TIMEOUT} --all) || + ( + echo "Failed to wait until databend operator is ready" && + kubectl get pods -n ${NAMESPACE} && + kubectl describe pods -n ${NAMESPACE} && + exit 1 + ) + +print_cluster_info() { + kubectl version + kubectl cluster-info + kubectl get nodes + kubectl get pods -n ${NAMESPACE} + kubectl describe pod -n ${NAMESPACE} +} + +print_cluster_info + +echo "Deploying meta cluster with helm" +helm repo add databend https://charts.databend.com +helm install meta databend/databend-meta --namespace ${NAMESPACE} + +(kubectl wait pods --for=condition=ready -n ${NAMESPACE} --timeout ${TIMEOUT} --all) || + ( + echo "Failed to wait until meta cluster is ready" && + kubectl get pods -n ${NAMESPACE} && + kubectl describe pods -n ${NAMESPACE} && + exit 1 + ) + +print_cluster_info diff --git a/scripts/gha/setup-cluster.sh b/scripts/gha/setup-kind-cluster.sh similarity index 100% rename from scripts/gha/setup-cluster.sh rename to scripts/gha/setup-kind-cluster.sh