From 24dc370be16173ce257ab7761cb3c2f578faaf07 Mon Sep 17 00:00:00 2001 From: Andre da Silva Date: Fri, 2 May 2025 14:06:48 -0300 Subject: [PATCH] Add Local SSD support --- kubernetes/linera-validator/helmfile.yaml | 25 +++- .../gke-daemonset-raid-disks.yaml | 68 ++++++++++ .../local-csi-driver/00_namespace.yaml | 4 + .../local-csi-driver/10_csidriver.yaml | 7 + .../10_driver_serviceaccount.yaml | 5 + .../10_provisioner_clusterrole.yaml | 100 ++++++++++++++ .../20_provisioner_clusterrolebinding.yaml | 12 ++ .../local-csi-driver/50_daemonset.yaml | 124 ++++++++++++++++++ .../scylla-setup/local-ssd-sc.yaml | 8 ++ .../linera-validator/scylla.values.yaml | 2 +- .../linera-validator/templates/shards.yaml | 3 + .../linera-validator/values-local.yaml.gotmpl | 1 + 12 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 kubernetes/linera-validator/scylla-setup/gke-daemonset-raid-disks.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-csi-driver/00_namespace.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-csi-driver/10_csidriver.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-csi-driver/10_driver_serviceaccount.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-csi-driver/10_provisioner_clusterrole.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-csi-driver/20_provisioner_clusterrolebinding.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-csi-driver/50_daemonset.yaml create mode 100644 kubernetes/linera-validator/scylla-setup/local-ssd-sc.yaml diff --git a/kubernetes/linera-validator/helmfile.yaml b/kubernetes/linera-validator/helmfile.yaml index 36651bbe470..163a5daeddb 100644 --- a/kubernetes/linera-validator/helmfile.yaml +++ b/kubernetes/linera-validator/helmfile.yaml @@ -3,6 +3,30 @@ environments: values: - writeToGrafanaCloud: {{ env "LINERA_WRITE_TO_GRAFANA_CLOUD" | default "false" }} validatorLabel: {{ env "LINERA_VALIDATOR_LABEL" | default (printf "local-%s" (env "USER")) }} + usingLocalSsd: {{ env "LINERA_HELMFILE_SET_USING_LOCAL_SSD" | default "false" }} + +helmDefaults: + wait: true + recreatePods: false + +{{- if .Values.usingLocalSsd }} +hooks: + - events: ["prepare"] + showlogs: true + command: bash + args: + - -c + - | + echo "Ensuring RAID0 and Local‑CSI driver are setup..." + + kubectl create -f ./scylla-setup/gke-daemonset-raid-disks.yaml + kubectl -n default rollout status daemonset/gke-raid-disks + + kubectl apply -f ./scylla-setup/local-csi-driver + kubectl -n local-csi-driver rollout status daemonset.apps/local-csi-driver + + kubectl apply -f ./scylla-setup/local-ssd-sc.yaml +{{- end }} --- @@ -69,4 +93,3 @@ releases: set: - name: crds.enabled value: "true" - diff --git a/kubernetes/linera-validator/scylla-setup/gke-daemonset-raid-disks.yaml b/kubernetes/linera-validator/scylla-setup/gke-daemonset-raid-disks.yaml new file mode 100644 index 00000000000..a7a72a23411 --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/gke-daemonset-raid-disks.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gke-raid-disks + namespace: default + labels: + k8s-app: gke-raid-disks +spec: + selector: + matchLabels: + name: gke-raid-disks + template: + metadata: + labels: + name: gke-raid-disks + spec: + nodeSelector: + cloud.google.com/gke-local-nvme-ssd: "true" + hostPID: true + containers: + - name: startup-script + image: registry.k8s.io/startup-script:v1 + securityContext: + privileged: true + env: + - name: STARTUP_SCRIPT + value: | + set -o errexit + set -o nounset + set -o pipefail + + # Ensure we have the XFS tools + if ! command -v mkfs.xfs >/dev/null; then + echo "mkfs.xfs not found! Installing xfsprogs..." + apt-get update + DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends xfsprogs + fi + + devices=() + for ssd in /dev/disk/by-id/google-local-ssd-block*; do + if [ -e "${ssd}" ]; then + devices+=("${ssd}") + fi + done + if [ "${#devices[@]}" -eq 0 ]; then + echo "No Local NVMe SSD disks found." + exit 1 + fi + + seen_arrays=(/dev/md/*) + device=${seen_arrays[0]} + echo "Setting RAID array with Local SSDs on device ${device}" + if [ ! -e "$device" ]; then + device="/dev/md/0" + echo "y" | mdadm --create "${device}" --level=0 --force --raid-devices=${#devices[@]} "${devices[@]}" + fi + + if ! blkid "${device}" >/dev/null 2>&1 ; then + echo "Formatting '${device}'" + mkfs.xfs -f "${device}" + fi + + mountpoint=/mnt/disks/raid + mkdir -p "${mountpoint}" + echo "Mounting '${device}' at '${mountpoint}'" + mount -o discard,prjquota,noatime,nodiratime "${device}" "${mountpoint}" + chmod a+w "${mountpoint}" \ No newline at end of file diff --git a/kubernetes/linera-validator/scylla-setup/local-csi-driver/00_namespace.yaml b/kubernetes/linera-validator/scylla-setup/local-csi-driver/00_namespace.yaml new file mode 100644 index 00000000000..56971bae6e5 --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-csi-driver/00_namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: local-csi-driver diff --git a/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_csidriver.yaml b/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_csidriver.yaml new file mode 100644 index 00000000000..b8c07f29dfa --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_csidriver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: local.csi.scylladb.com +spec: + attachRequired: false + storageCapacity: true diff --git a/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_driver_serviceaccount.yaml b/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_driver_serviceaccount.yaml new file mode 100644 index 00000000000..9a5ca4e31f2 --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_driver_serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-csi-driver + namespace: local-csi-driver diff --git a/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_provisioner_clusterrole.yaml b/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_provisioner_clusterrole.yaml new file mode 100644 index 00000000000..c211a270b2f --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-csi-driver/10_provisioner_clusterrole.yaml @@ -0,0 +1,100 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scylladb:csi-external-provisioner +rules: +- apiGroups: + - "" + resources: + - "persistentvolumes" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "delete" +- apiGroups: + - "" + resources: + - "persistentvolumeclaims" + verbs: + - "get" + - "list" + - "watch" + - "update" +- apiGroups: + - "storage.k8s.io" + resources: + - "storageclasses" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "" + resources: + - "events" + verbs: + - "list" + - "watch" + - "create" + - "update" + - "patch" +- apiGroups: + - "snapshot.storage.k8s.io" + resources: + - "volumesnapshots" + verbs: + - "get" + - "list" +- apiGroups: + - "snapshot.storage.k8s.io" + resources: + - "volumesnapshotcontents" + verbs: + - "get" + - "list" +- apiGroups: + - "storage.k8s.io" + resources: + - "csinodes" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "" + resources: + - "nodes" + verbs: + - "get" + - "list" + - "watch" +- apiGroups: + - "storage.k8s.io" + resources: + - "csistoragecapacities" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" +# The GET permissions below are needed for walking up the ownership chain +# for CSIStorageCapacity. They are sufficient for deployment via +# StatefulSet (only needs to get Pod) and Deployment (needs to get +# Pod and then ReplicaSet to find the Deployment). +- apiGroups: + - "" + resources: + - "pods" + verbs: + - "get" +- apiGroups: + - "apps" + resources: + - "replicasets" + verbs: + - "get" diff --git a/kubernetes/linera-validator/scylla-setup/local-csi-driver/20_provisioner_clusterrolebinding.yaml b/kubernetes/linera-validator/scylla-setup/local-csi-driver/20_provisioner_clusterrolebinding.yaml new file mode 100644 index 00000000000..f86956a8259 --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-csi-driver/20_provisioner_clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scylladb:csi-external-provisioner +subjects: +- kind: ServiceAccount + name: local-csi-driver + namespace: local-csi-driver +roleRef: + kind: ClusterRole + name: scylladb:csi-external-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/kubernetes/linera-validator/scylla-setup/local-csi-driver/50_daemonset.yaml b/kubernetes/linera-validator/scylla-setup/local-csi-driver/50_daemonset.yaml new file mode 100644 index 00000000000..27a7b5757ce --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-csi-driver/50_daemonset.yaml @@ -0,0 +1,124 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: local-csi-driver + namespace: local-csi-driver + labels: + app.kubernetes.io/name: local-csi-driver +spec: + selector: + matchLabels: + app.kubernetes.io/name: local-csi-driver + template: + metadata: + labels: + app.kubernetes.io/name: local-csi-driver + spec: + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: local-csi-driver + tolerations: + - operator: Exists + containers: + - name: local-csi-driver + securityContext: + privileged: true + image: docker.io/scylladb/local-csi-driver:latest + imagePullPolicy: IfNotPresent + args: + - --listen=/csi/csi.sock + - --node-name=$(NODE_NAME) + - --volumes-dir=/mnt/disks/raid + - --v=2 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: volumes-dir + mountPath: /mnt/disks/raid + ports: + - name: healthz + containerPort: 9809 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + - name: csi-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar@sha256:fdff3ee285341bc58033b6b2458a5d45fd90ec6922a8ba6ebdd49b0c41e2cd34 + imagePullPolicy: IfNotPresent + args: + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/local.csi.scylladb.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe@sha256:cacee2b5c36dd59d4c7e8469c05c9e4ef53ecb2df9025fa8c10cdaf61bce62f0 + imagePullPolicy: IfNotPresent + args: + - --csi-address=/csi/csi.sock + - --health-port=9809 + - --v=2 + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner@sha256:ee3b525d5b89db99da3b8eb521d9cd90cb6e9ef0fbb651e98bb37be78d36b5b8 + imagePullPolicy: IfNotPresent + args: + - --csi-address=/csi/csi.sock + - --v=2 + - --node-deployment + - --feature-gates=Topology=true + - --immediate-topology=false + - --enable-capacity + - --capacity-ownerref-level=0 + - --capacity-poll-interval=30s + - --default-fstype=xfs + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/local.csi.scylladb.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: volumes-dir + hostPath: + path: /mnt/disks/raid + type: Directory diff --git a/kubernetes/linera-validator/scylla-setup/local-ssd-sc.yaml b/kubernetes/linera-validator/scylla-setup/local-ssd-sc.yaml new file mode 100644 index 00000000000..434055c0866 --- /dev/null +++ b/kubernetes/linera-validator/scylla-setup/local-ssd-sc.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: nvme-ssd-block +provisioner: local.csi.scylladb.com +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +allowVolumeExpansion: false diff --git a/kubernetes/linera-validator/scylla.values.yaml b/kubernetes/linera-validator/scylla.values.yaml index 7861de82ab4..98f8c3add4f 100644 --- a/kubernetes/linera-validator/scylla.values.yaml +++ b/kubernetes/linera-validator/scylla.values.yaml @@ -3,7 +3,7 @@ sysctls: - "fs.aio-max-nr=4082080" datacenter: validator racks: - - name: rack-1 + - name: rack members: 1 scyllaConfig: "scylla-config" storage: diff --git a/kubernetes/linera-validator/templates/shards.yaml b/kubernetes/linera-validator/templates/shards.yaml index 5ee113f99d5..5ce4954dc8d 100644 --- a/kubernetes/linera-validator/templates/shards.yaml +++ b/kubernetes/linera-validator/templates/shards.yaml @@ -80,6 +80,9 @@ spec: name: linera-db spec: accessModes: ["ReadWriteOnce"] + {{- if .Values.usingLocalSsd }} + storageClassName: nvme-ssd-block + {{- end }} resources: requests: storage: {{ .Values.rocksdbStorageSize }} diff --git a/kubernetes/linera-validator/values-local.yaml.gotmpl b/kubernetes/linera-validator/values-local.yaml.gotmpl index 873d6a1b902..f9cf788329c 100644 --- a/kubernetes/linera-validator/values-local.yaml.gotmpl +++ b/kubernetes/linera-validator/values-local.yaml.gotmpl @@ -11,6 +11,7 @@ numShards: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }} rocksdbStorageSize: {{ env "LINERA_HELMFILE_SET_ROCKSDB_STORAGE_SIZE" | default "2Gi" }} storage: {{ env "LINERA_HELMFILE_SET_STORAGE" | default "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042" }} dualStore: {{ env "LINERA_HELMFILE_SET_DUAL_STORE" | default "false" }} +usingLocalSsd: {{ env "LINERA_HELMFILE_SET_USING_LOCAL_SSD" | default "false" }} # Loki loki-stack: