Merge pull request #160 from nebius/feature/storage-attachment

dstaroff · web-flow · commit f0f950da6166 · 2024-11-06T15:47:20.000+01:00
add: arrange storage attachment scheduling by storage type
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.14.16
+1.14.17
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -3,4 +3,4 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/slurm-operator
-    newTag: 1.14.16
+    newTag: 1.14.17
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -82,7 +82,7 @@ spec:
             value: "false"
           - name: SLURM_OPERATOR_WATCH_NAMESPACES
             value: "*"
-        image: controller:1.14.16
+        image: controller:1.14.17
         imagePullPolicy: Always
         name: manager
         securityContext:
diff --git a/helm/slurm-cluster-storage/Chart.yaml b/helm/slurm-cluster-storage/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-slurm-cluster-storage
 description: A Helm chart for Kubernetes
 type: application
-version: "1.14.16"
-appVersion: "1.14.16"
+version: "1.14.17"
+appVersion: "1.14.17"
diff --git a/helm/slurm-cluster-storage/templates/accounting-mount-daemonset.yaml b/helm/slurm-cluster-storage/templates/accounting-mount-daemonset.yaml
@@ -41,7 +41,7 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-{{ .Values.scheduling.accounting.matchExpressions | toYaml | indent 18 }}
+{{ .Values.storage.accounting.matchExpressions | toYaml | indent 18 }}
       tolerations:
-{{ .Values.scheduling.accounting.tolerations | toYaml | indent 8 }}
+{{ .Values.storage.accounting.tolerations | toYaml | indent 8 }}
 {{- end }}
diff --git a/helm/slurm-cluster-storage/templates/accounting-pv.yaml b/helm/slurm-cluster-storage/templates/accounting-pv.yaml
@@ -20,5 +20,5 @@ spec:
     required:
       nodeSelectorTerms:
         - matchExpressions:
-{{ .Values.scheduling.accounting.matchExpressions | toYaml | indent 12 }}
+{{ .Values.storage.accounting.matchExpressions | toYaml | indent 12 }}
 {{- end }}
diff --git a/helm/slurm-cluster-storage/templates/controller-spool-mount-daemonset.yaml b/helm/slurm-cluster-storage/templates/controller-spool-mount-daemonset.yaml
@@ -40,6 +40,6 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-{{ .Values.scheduling.controller.matchExpressions | toYaml | indent 18 }}
+{{ .Values.storage.controllerSpool.matchExpressions | toYaml | indent 18 }}
       tolerations:
-{{ .Values.scheduling.controller.tolerations | toYaml | indent 8 }}
+{{ .Values.storage.controllerSpool.tolerations | toYaml | indent 8 }}
diff --git a/helm/slurm-cluster-storage/templates/controller-spool-pv.yaml b/helm/slurm-cluster-storage/templates/controller-spool-pv.yaml
@@ -22,4 +22,4 @@ spec:
     required:
       nodeSelectorTerms:
         - matchExpressions:
-{{ .Values.scheduling.controller.matchExpressions | toYaml | indent 12 }}
+{{ .Values.storage.controllerSpool.matchExpressions | toYaml | indent 12 }}
diff --git a/helm/slurm-cluster-storage/templates/jail-mount-daemonset.yaml b/helm/slurm-cluster-storage/templates/jail-mount-daemonset.yaml
@@ -53,6 +53,6 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-{{ .Values.scheduling.worker.matchExpressions | toYaml | indent 18 }}
+{{ .Values.storage.jail.matchExpressions | toYaml | indent 18 }}
       tolerations:
-{{ .Values.scheduling.worker.tolerations | toYaml | indent 8 }}
+{{ .Values.storage.jail.tolerations | toYaml | indent 8 }}
diff --git a/helm/slurm-cluster-storage/templates/jail-pv.yaml b/helm/slurm-cluster-storage/templates/jail-pv.yaml
@@ -24,4 +24,4 @@ spec:
     required:
       nodeSelectorTerms:
         - matchExpressions:
-{{ .Values.scheduling.worker.matchExpressions | toYaml | indent 12 }}
+{{ .Values.storage.jail.matchExpressions | toYaml | indent 12 }}
diff --git a/helm/slurm-cluster-storage/templates/jail-submounts-mount-daemonset.yaml b/helm/slurm-cluster-storage/templates/jail-submounts-mount-daemonset.yaml
@@ -41,8 +41,8 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-{{ $.Values.scheduling.worker.matchExpressions | toYaml | indent 18 }}
+{{ $.Values.storage.jail.matchExpressions | toYaml | indent 18 }}
       tolerations:
-{{ $.Values.scheduling.worker.tolerations | toYaml | indent 8}}
+{{ $.Values.storage.jail.tolerations | toYaml | indent 8}}
 ---
 {{- end }}
diff --git a/helm/slurm-cluster-storage/templates/jail-submounts-pv.yaml b/helm/slurm-cluster-storage/templates/jail-submounts-pv.yaml
@@ -24,6 +24,6 @@ spec:
     required:
       nodeSelectorTerms:
         - matchExpressions:
-{{ $.Values.scheduling.worker.matchExpressions | toYaml | indent 12 }}
+{{ $.Values.storage.jail.matchExpressions | toYaml | indent 12 }}
 ---
 {{- end }}
diff --git a/helm/slurm-cluster-storage/values.yaml b/helm/slurm-cluster-storage/values.yaml
@@ -28,7 +28,7 @@ volume:
   #     size: "1500Gi"
   #     filestoreDeviceName: "mlperf-sd"
 
-scheduling:
+storage:
   accounting:
     matchExpressions: []
     # matchExpressions:
@@ -42,7 +42,7 @@ scheduling:
     #     operator: Exists
     #     effect: NoSchedule
 
-  controller:
+  controllerSpool:
     matchExpressions: []
     # matchExpressions:
     #   - key: nebius.com/node-group-id
@@ -55,7 +55,7 @@ scheduling:
     #     operator: Exists
     #     effect: NoSchedule
 
-  worker:
+  jail:
     matchExpressions: []
     # matchExpressions:
     #   - key: nebius.com/node-group-id
diff --git a/helm/slurm-cluster/Chart.yaml b/helm/slurm-cluster/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-slurm-cluster
 description: A Helm chart for Kubernetes
 type: application
-version: "1.14.16"
-appVersion: "1.14.16"
+version: "1.14.17"
+appVersion: "1.14.17"
diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml
@@ -369,12 +369,12 @@ telemetry: {}
 #   otelCollectorPort: 8429
 
 images:
-  slurmctld: "cr.eu-north1.nebius.cloud/soperator/controller_slurmctld:1.14.16-jammy-slurm24.05.2"
-  slurmd: "cr.eu-north1.nebius.cloud/soperator/worker_slurmd:1.14.16-jammy-slurm24.05.2"
-  sshd: "cr.eu-north1.nebius.cloud/soperator/login_sshd:1.14.16-jammy-slurm24.05.2"
-  munge: "cr.eu-north1.nebius.cloud/soperator/munge:1.14.16-jammy-slurm24.05.2"
-  populateJail: "cr.eu-north1.nebius.cloud/soperator/populate_jail:1.14.16-jammy-slurm24.05.2"
-  ncclBenchmark: "cr.eu-north1.nebius.cloud/soperator/nccl_benchmark:1.14.16-jammy-slurm24.05.2"
-  slurmdbd: "cr.eu-north1.nebius.cloud/soperator/controller_slurmdbd:1.14.16-jammy-slurm24.05.2"
-  exporter: "cr.eu-north1.nebius.cloud/soperator/exporter:1.14.16-jammy-slurm24.05.2"
+  slurmctld: "cr.eu-north1.nebius.cloud/soperator/controller_slurmctld:1.14.17-jammy-slurm24.05.2"
+  slurmd: "cr.eu-north1.nebius.cloud/soperator/worker_slurmd:1.14.17-jammy-slurm24.05.2"
+  sshd: "cr.eu-north1.nebius.cloud/soperator/login_sshd:1.14.17-jammy-slurm24.05.2"
+  munge: "cr.eu-north1.nebius.cloud/soperator/munge:1.14.17-jammy-slurm24.05.2"
+  populateJail: "cr.eu-north1.nebius.cloud/soperator/populate_jail:1.14.17-jammy-slurm24.05.2"
+  ncclBenchmark: "cr.eu-north1.nebius.cloud/soperator/nccl_benchmark:1.14.17-jammy-slurm24.05.2"
+  slurmdbd: "cr.eu-north1.nebius.cloud/soperator/controller_slurmdbd:1.14.17-jammy-slurm24.05.2"
+  exporter: "cr.eu-north1.nebius.cloud/soperator/exporter:1.14.17-jammy-slurm24.05.2"
   mariaDB: "docker-registry1.mariadb.com/library/mariadb:11.4.3"
diff --git a/helm/soperator-crds/Chart.yaml b/helm/soperator-crds/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-soperator-crds
 description: A Helm chart for Kubernetes
 type: application
-version: 1.14.16
-appVersion: "1.14.16"
+version: 1.14.17
+appVersion: "1.14.17"
diff --git a/helm/soperator/Chart.yaml b/helm/soperator/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-soperator
 description: A Helm chart for Kubernetes
 type: application
-version: 1.14.16
-appVersion: "1.14.16"
+version: 1.14.17
+appVersion: "1.14.17"
diff --git a/helm/soperator/templates/deployment.yaml b/helm/soperator/templates/deployment.yaml
@@ -75,7 +75,7 @@ spec:
           }}
         securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext
           | nindent 10 }}
-      securityContext:
-        runAsNonRoot: true
+      securityContext: {{- toYaml .Values.controllerManager.podSecurityContext | nindent
+        8 }}
       serviceAccountName: {{ include "soperator.fullname" . }}-controller-manager
       terminationGracePeriodSeconds: 10
diff --git a/helm/soperator/values.yaml b/helm/soperator/values.yaml
@@ -36,14 +36,16 @@ controllerManager:
       slurmOperatorWatchNamespaces: '*'
     image:
       repository: cr.eu-north1.nebius.cloud/soperator/slurm-operator
-      tag: 1.14.16
+      tag: 1.14.17
     imagePullPolicy: Always
     resources:
       limits:
         memory: 128Mi
       requests:
         cpu: 100m
         memory: 64Mi
+  podSecurityContext:
+    runAsNonRoot: true
   replicas: 1
   serviceAccount:
     annotations: {}
diff --git a/images/worker/slurmd_entrypoint.sh b/images/worker/slurmd_entrypoint.sh
@@ -4,16 +4,12 @@ set -e # Exit immediately if any command returns a non-zero error code
 
 echo "Starting slurmd entrypoint script"
 if [ -n "${CGROUP_V2}" ]; then
-    CGROUP_PATH=''
-    if [ "$SLURM_CLUSTER_TYPE" = "gpu" ]; then
-        CGROUP_PATH=$(cat /proc/self/cgroup | awk -F'/' '{print "/"$2"/"$3"/"$4}')
-    else
-        CGROUP_PATH=$(cat /proc/self/cgroup | awk -F'/' '{print "/"$2"/"$3}')
-    fi
+    CGROUP_PATH=$(cat /proc/self/cgroup | awk -F'/' '{print "/"$2"/"$3"/"$4}')
 
     if [ -n "${CGROUP_PATH}" ]; then
         echo "cgroup v2 detected, creating cgroup for ${CGROUP_PATH}"
         mkdir -p /sys/fs/cgroup/${CGROUP_PATH}/system.slice
+        mkdir -p /sys/fs/cgroup/${CGROUP_PATH}/../system.slice
     else
         echo "cgroup v2 detected, but cgroup path is empty"
         exit 1
diff --git a/internal/consts/version.go b/internal/consts/version.go
@@ -2,5 +2,5 @@
 package consts
 
 const (
-	VersionCR = "1.14.16"
+	VersionCR = "1.14.17"
 )
diff --git a/internal/render/accounting/mariadb.go b/internal/render/accounting/mariadb.go
@@ -10,7 +10,7 @@ import (
 	"k8s.io/utils/ptr"
 
 	slurmv1 "nebius.ai/slurm-operator/api/v1"
-	consts "nebius.ai/slurm-operator/internal/consts"
+	"nebius.ai/slurm-operator/internal/consts"
 	"nebius.ai/slurm-operator/internal/naming"
 	"nebius.ai/slurm-operator/internal/render/common"
 	"nebius.ai/slurm-operator/internal/utils"
@@ -93,6 +93,7 @@ func RenderMariaDb(
 			PodTemplate: mariadv1alpha1.PodTemplate{
 				NodeSelector:       nodeFilter.NodeSelector,
 				Affinity:           affinityConfig,
+				Tolerations:        nodeFilter.Tolerations,
 				PodSecurityContext: mariaDb.PodSecurityContext,
 			},
 			Metrics: mariaDb.Metrics,
diff --git a/internal/render/accounting/mariadb_test.go b/internal/render/accounting/mariadb_test.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
+
 	slurmv1 "nebius.ai/slurm-operator/api/v1"
 	"nebius.ai/slurm-operator/internal/consts"
 	"nebius.ai/slurm-operator/internal/values"
