Merge pull request #389 from nebius/many-small-changes-aggregated/1

Pre-create enroot credentials, exclude enroot bind-mounts from motd, make Docker mount /dev/infiniband, store /tmp on disk and add tmpfs /mnt/memory

Author: rdjjke

VERSION

@@ -1 +1 @@
-1.17.0
+1.18.0

api/v1/slurmcluster_types.go

@@ -86,6 +86,13 @@ type SlurmClusterSpec struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default={defMemPerNode: 1228800, defCpuPerGPU: 16, completeWait: 5, debugFlags: "Cgroup,CPU_Bind,Gres,JobComp,Priority,Script,SelectType,Steps,TraceJobs", epilog: "", prolog: "", taskPluginParam: "", maxJobCount: 10000, minJobAge: 86400}
 	SlurmConfig SlurmConfig `json:"slurmConfig,omitempty"`
+
+	// MPIConfig represents the PMIx configuration in mpi.conf. Not all options are supported.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default={pmixEnv: "OMPI_MCA_btl_tcp_if_include=eth0"}
+	MPIConfig MPIConfig `json:"mpiConfig,omitempty"`
+
 	// Generate and set default AppArmor profile for the Slurm worker and login nodes. The Security Profiles Operator must be installed.
 	//
 	// +kubebuilder:default=false
@@ -143,6 +150,16 @@ type SlurmConfig struct {
 	MinJobAge *int32 `json:"minJobAge,omitempty"`
 }
 
+type MPIConfig struct {
+	// Semicolon separated list of environment variables to be set in job environments to be used by PMIx.
+	// Defaults to "OMPI_MCA_btl_tcp_if_include=eth0" to avoid "lo" and "docker" interfaces to be selected by OpenMPI.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default="OMPI_MCA_btl_tcp_if_include=eth0"
+	// +kubebuilder:validation:Optional
+	PMIxEnv string `json:"pmixEnv,omitempty"`
+}
+
 type PartitionConfiguration struct {
 	// ConfigType
 	// +kubebuilder:validation:Enum=default;custom
@@ -319,7 +336,8 @@ type NCCLArguments struct {
 	// +kubebuilder:default="0"
 	ThresholdMoreThan string `json:"thresholdMoreThan,omitempty"`
 
-	// UseInfiniband defines using NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 NCCL_ALGO=Ring env variables for test
+	// UseInfiniband defines using NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 NCCL_ALGO=Ring env variables for test.
+	// According to NVIDIA these env vars should be used only for debugging.
 	// https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html
 	//
 	// +kubebuilder:validation:Optional

api/v1/zz_generated.deepcopy.go

@@ -1082,6 +1082,19 @@ spec:
                 - downscaleAndOverwritePopulateJail
                 - skipPopulateJail
                 type: string
+              mpiConfig:
+                default:
+                  pmixEnv: OMPI_MCA_btl_tcp_if_include=eth0
+                description: MPIConfig represents the PMIx configuration in mpi.conf.
+                  Not all options are supported.
+                properties:
+                  pmixEnv:
+                    default: OMPI_MCA_btl_tcp_if_include=eth0
+                    description: |-
+                      Semicolon separated list of environment variables to be set in job environments to be used by PMIx.
+                      Defaults to "OMPI_MCA_btl_tcp_if_include=eth0" to avoid "lo" and "docker" interfaces to be selected by OpenMPI.
+                    type: string
+                type: object
               ncclSettings:
                 description: NCCLSettings
                 properties:
@@ -1205,7 +1218,8 @@ spec:
                           useInfiniband:
                             default: true
                             description: |-
-                              UseInfiniband defines using NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 NCCL_ALGO=Ring env variables for test
+                              UseInfiniband defines using NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 NCCL_ALGO=Ring env variables for test.
+                              According to NVIDIA these env vars should be used only for debugging.
                               https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html
                             type: boolean
                         type: object
@@ -1487,9 +1501,9 @@ spec:
                   defCpuPerGPU: 16
                   defMemPerNode: 1228800
                   epilog: ""
-                  prolog: ""
                   maxJobCount: 10000
                   minJobAge: 86400
+                  prolog: ""
                   taskPluginParam: ""
                 description: SlurmConfig represents the Slurm configuration in slurm.conf.
                   Not all options are supported.
@@ -1519,11 +1533,8 @@ spec:
                     type: integer
                   epilog:
                     default: ""
-                    description: The Epilog script runs after a job completes
-                    type: string
-                  prolog:
-                    default: ""
-                    description: The Prolog script runs before a job starts on the compute node
+                    description: Defines specific file to run the epilog when job
+                      ends. Default value is no epilog
                     type: string
                   maxJobCount:
                     default: 10000
@@ -1536,6 +1547,11 @@ spec:
                       time
                     format: int32
                     type: integer
+                  prolog:
+                    default: ""
+                    description: Defines specific file to run the prolog when job
+                      starts. Default value is no prolog
+                    type: string
                   taskPluginParam:
                     default: ""
                     description: Additional parameters for the task plugin

config/crd/bases/slurm.nebius.ai_slurmclusters.yaml

@@ -3,7 +3,7 @@
 # It should be run by config/default
 resources:
 - bases/slurm.nebius.ai_slurmclusters.yaml
-- bases/slurm.nebius.ai_nodeconfigurators.yaml
+#- bases/slurm.nebius.ai_nodeconfigurators.yaml
 
 #+kubebuilder:scaffold:crdkustomizeresource
 

config/crd/kustomization.yaml

@@ -3,4 +3,4 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/slurm-operator
-    newTag: 1.17.0
+    newTag: 1.18.0

config/manager/kustomization.yaml

@@ -87,7 +87,7 @@ spec:
             value: "false"
           - name: SLURM_OPERATOR_WATCH_NAMESPACES
             value: "*"
-        image: controller:1.17.0
+        image: controller:1.18.0
         imagePullPolicy: Always
         name: manager
         securityContext:

config/manager/manager.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: helm-slurm-cluster-storage
 description: A Helm chart for Kubernetes
 type: application
-version: "1.17.0"
-appVersion: "1.17.0"
+version: "1.18.0"
+appVersion: "1.18.0"

helm/slurm-cluster-storage/Chart.yaml

@@ -3,6 +3,8 @@ kind: PersistentVolumeClaim
 metadata:
   namespace: {{ .Release.Namespace }}
   name: {{ include "slurm-cluster-storage.volume.jail.pvc" . }}
+  annotations:
+    k8up.io/backup: 'true'
 spec:
   storageClassName: {{ include "slurm-cluster-storage.volume.jail.storageClass" . }}
   resources:

helm/slurm-cluster-storage/templates/jail-pvc.yaml

@@ -2,6 +2,6 @@ apiVersion: v2
 name: helm-slurm-cluster
 description: A Helm chart for Kubernetes
 type: application
-version: "1.17.0"
-appVersion: "1.17.0"
+version: "1.18.0"
+appVersion: "1.18.0"
 kubeVersion: ">=1.29.0-0"