Merge pull request #90 from xrstf/byenami

Use upstream approach and images to installing etcd

Author: kcp-ci-bot

Makefile

@@ -8,6 +8,7 @@ KUSTOMIZE_VERSION ?= v5.4.3
 CONTROLLER_TOOLS_VERSION ?= v0.16.1
 GOLANGCI_LINT_VERSION ?= 2.1.6
 PROTOKOL_VERSION ?= 0.7.2
+HELM_VERSION ?= 3.18.6
 
 # Image URL to use all building/pushing image targets
 IMG ?= ghcr.io/kcp-dev/kcp-operator
@@ -153,6 +154,7 @@ GOLANGCI_LINT = $(TOOLS_DIR)/golangci-lint
 PROTOKOL = $(TOOLS_DIR)/protokol
 RECONCILER_GEN := $(TOOLS_DIR)/reconciler-gen
 OPENSHIFT_GOIMPORTS := $(TOOLS_DIR)/openshift-goimports
+HELM := $(TOOLS_DIR)/helm
 
 .PHONY: kubectl
 kubectl: $(KUBECTL) ## Download kubectl locally if necessary.
@@ -196,6 +198,13 @@ openshift-goimports: $(OPENSHIFT_GOIMPORTS) ## Download openshift-goimports loca
 $(OPENSHIFT_GOIMPORTS):
 	@GO_MODULE=true hack/download-tool.sh github.com/openshift-eng/openshift-goimports openshift-goimports $(OPENSHIFT_GOIMPORTS_VER)
 
+.PHONY: helm
+helm: $(HELM) ## Download Helm locally if necessary.
+
+.PHONY: $(HELM)
+$(HELM):
+	@hack/download-tool.sh https://get.helm.sh/helm-v${HELM_VERSION}-$(shell go env GOOS)-$(shell go env GOARCH).tar.gz helm $(HELM_VERSION)
+
 ##@ Documentation
 
 VENVDIR=$(abspath docs/venv)

docs/content/setup/quickstart.md

@@ -12,10 +12,11 @@ kcp-operator has to be installed according to the instructions given in [Setup](
 !!! warning
     Never deploy etcd like below in production as it sets up an etcd instance without authentication or TLS.
 
-Running a root shard requires a running etcd instance/cluster. A simple one can be set up with Helm and the Bitnami etcd chart:
+Running a root shard requires a running etcd instance/cluster. A simple one can be set up with Helm and the CI chart used by the kcp-operator for testing itself:
 
 ```sh
-helm install etcd oci://registry-1.docker.io/bitnamicharts/etcd --set auth.rbac.enabled=false --set auth.rbac.create=false
+git clone https://github.com/kcp-dev/kcp-operator
+helm install etcd ./kcp-operator/hack/ci/testdata/etcd
 ```
 
 ## Create Root Shard
@@ -85,7 +86,7 @@ spec:
     spec:
       # expose this front-proxy via a load balancer
       type: LoadBalancer
-``` 
+```
 
 kcp-operator will deploy a kcp-front-proxy installation based on this and connect it to the `root` root shard created before.
 
@@ -166,7 +167,7 @@ To create a workspace, run:
 
 ```sh
 kubectl create-workspace test
-``` 
+```
 
 Output should look like this:
 

hack/ci/run-e2e-tests.sh

@@ -67,6 +67,9 @@ chmod 600 "$KUBECONFIG"
 make protokol
 _tools/protokol --output "$ARTIFACTS/logs" --namespace 'kcp-*' --namespace 'e2e-*' >/dev/null 2>&1 &
 
+# need Helm to setup etcd
+make helm
+
 # load the operator image into the kind cluster
 image="ghcr.io/kcp-dev/kcp-operator:$IMAGE_TAG"
 archive=operator.tar
@@ -84,10 +87,10 @@ kubectl --namespace kcp-operator-system wait pod --all --for condition=Ready
 # deploying cert-manager
 echo "Deploying cert-manager…"
 
-helm repo add jetstack https://charts.jetstack.io --force-update
-helm repo update
+_tools/helm repo add jetstack https://charts.jetstack.io --force-update
+_tools/helm repo update
 
-helm upgrade \
+_tools/helm upgrade \
   --install \
   --namespace cert-manager \
   --create-namespace \
@@ -98,6 +101,10 @@ helm upgrade \
 kubectl apply --filename hack/ci/testdata/clusterissuer.yaml
 
 echo "Running e2e tests…"
+
+export HELM_BINARY="$(realpath _tools/helm)"
+export ETCD_HELM_CHART="$(realpath hack/ci/testdata/etcd)"
+
 (set -x; go test -tags e2e -timeout 2h -v ./test/e2e/...)
 
 echo "Done. :-)"

hack/ci/testdata/etcd/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: etcd
+version: 1.0.0
+appVersion: "3.5.21"
+sources:
+  - https://etcd.io/docs/v3.5/op-guide/kubernetes/

hack/ci/testdata/etcd/templates/serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: '{{ .Release.Name }}'
+automountServiceAccountToken: false

hack/ci/testdata/etcd/templates/services.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-headless'
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+spec:
+  type: ClusterIP
+  clusterIP: None
+  publishNotReadyAddresses: true
+  selector:
+    app.kubernetes.io/instance: '{{ .Release.Name }}'
+    app.kubernetes.io/name: etcd
+  ports:
+    - name: client
+      port: 2379
+      targetPort: client
+    - name: peer
+      port: 2380
+      targetPort: peer
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}'
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/instance: '{{ .Release.Name }}'
+    app.kubernetes.io/name: etcd
+  ports:
+    - name: client
+      port: 2379
+      targetPort: client
+      nodePort: null
+    - name: peer
+      port: 2380
+      targetPort: peer
+      nodePort: null

hack/ci/testdata/etcd/templates/statefulset.yaml

@@ -0,0 +1,108 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}'
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: '{{ .Release.Name }}'
+      app.kubernetes.io/name: etcd
+  serviceName: '{{ .Release.Name }}-headless'
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: '{{ .Release.Name }}'
+        app.kubernetes.io/name: etcd
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        fsGroup: 1001
+        fsGroupChangePolicy: Always
+      serviceAccountName: '{{ .Release.Name }}'
+      containers:
+        - name: etcd
+          image: quay.io/coreos/etcd:v3.5.21
+          imagePullPolicy: IfNotPresent
+          command:
+            - /usr/local/bin/etcd
+          args:
+            - --name=$(HOSTNAME)
+            - --data-dir=/data
+            - --wal-dir=/data/wal
+            - --listen-peer-urls=$(URI_SCHEME)://0.0.0.0:2380
+            - --listen-client-urls=$(URI_SCHEME)://0.0.0.0:2379
+            - --advertise-client-urls=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME).svc.cluster.local:2379
+            - --initial-cluster-state=new
+            - --initial-cluster-token=$(HOSTNAME)
+            - --initial-cluster=$(HOSTNAME)=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME).svc.cluster.local:2380
+            - --initial-advertise-peer-urls=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME).svc.cluster.local:2380
+            - --listen-metrics-urls=http://0.0.0.0:8080
+          env:
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SERVICE_NAME
+              value: '{{ .Release.Name }}-headless'
+            - name: ETCDCTL_ENDPOINTS
+              value: $(HOSTNAME).$(SERVICE_NAME):2379
+            - name: URI_SCHEME
+              value: "http"
+          ports:
+            - name: client
+              containerPort: 2379
+              protocol: TCP
+            - name: peer
+              containerPort: 2380
+              protocol: TCP
+            - name: metrics
+              containerPort: 8080
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: metrics
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: metrics
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 30
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsGroup: 1001
+            runAsNonRoot: true
+            runAsUser: 1001
+            seccompProfile:
+              type: RuntimeDefault
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 8Gi

hack/ci/testdata/etcd/values.yaml

@@ -32,18 +32,17 @@ import (
 func DeployEtcd(t *testing.T, name, namespace string) string {
 	t.Helper()
 
+	helmChart := os.Getenv("ETCD_HELM_CHART")
+
 	t.Logf("Installing etcd %q into %s…", name, namespace)
-	args := []string{
-		"install",
-		name,
-		"oci://registry-1.docker.io/bitnamicharts/etcd",
-		"--namespace", namespace,
-		"--version", "10.7.1", // latest version at the time of writing
-		"--set", "auth.rbac.enabled=false",
-		"--set", "auth.rbac.create=false",
+	args := []string{"install", "--namespace", namespace, "--atomic", name, helmChart}
+
+	helmCommand := os.Getenv("HELM_BINARY")
+	if helmCommand == "" {
+		helmCommand = "helm"
 	}
 
-	if err := exec.Command("helm", args...).Run(); err != nil {
+	if err := exec.Command(helmCommand, args...).Run(); err != nil {
 		t.Fatalf("Failed to deploy etcd: %v", err)
 	}
 
@@ -57,7 +56,12 @@ func DeployEtcd(t *testing.T, name, namespace string) string {
 		"--timeout", "3m",
 	}
 
-	if err := exec.Command("kubectl", args...).Run(); err != nil {
+	kubectlCommand := os.Getenv("KUBECTL_BINARY")
+	if kubectlCommand == "" {
+		kubectlCommand = "kubectl"
+	}
+
+	if err := exec.Command(kubectlCommand, args...).Run(); err != nil {
 		t.Fatalf("Failed to wait for etcd to become ready: %v", err)
 	}