Merge pull request #1171 from nebius/dev

Release soperator 1.21.5

Author: theyoprst

VERSION

@@ -1 +1 @@
-1.21.4
+1.21.5

api/v1/slurmcluster_types.go

@@ -105,7 +105,7 @@ type SlurmClusterSpec struct {
 	// PlugStackConfig represents the Plugin stack configurations in `plugstack.conf`.
 	//
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default={ pyxis: { required: true, containerImageSave: "/var/cache/enroot-container-images/" }, ncclDebug: { required: false, enabled: false, logLevel: "INFO", outputToFile: true, outputToStdOut: false, outputDirectory: "/opt/soperator-outputs/%h/nccl_logs" } }
+	// +kubebuilder:default={ pyxis: { required: true, containerImageSave: "/var/cache/enroot-container-images/" }, ncclDebug: { required: false, enabled: false, logLevel: "INFO", outputToFile: true, outputToStdOut: false, outputDirectory: "/opt/soperator-outputs/nccl_logs" } }
 	PlugStackConfig PlugStackConfig `json:"plugStackConfig,omitempty"`
 
 	// SlurmTopologyConfigMapRefName is the name of the slurm topology config.
@@ -215,7 +215,7 @@ type PlugStackConfig struct {
 	// NcclDebug represents the 'NCCL Debug' SPANK plugin configuration.
 	//
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default={ required: false, enabled: false, logLevel: "INFO", outputToFile: true, outputToStdOut: false, outputDirectory: "/opt/soperator-outputs/%h/nccl_logs" }
+	// +kubebuilder:default={ required: false, enabled: false, logLevel: "INFO", outputToFile: true, outputToStdOut: false, outputDirectory: "/opt/soperator-outputs/nccl_logs" }
 	NcclDebug PluginConfigNcclDebug `json:"ncclDebug,omitempty"`
 
 	// PluginConfigCustom represents a configuration of custom SPANK plugins.
@@ -271,7 +271,7 @@ type PluginConfigNcclDebug struct {
 
 	// OutputToFile defines whether to additionally redirect `NCCL_DEBUG` outputs to the output file.
 	// Output filename will have the following format:
-	//  <JOB_ID>.<STEP_ID>.out
+	//  <WORKER_NAME>.<JOB_ID>.<STEP_ID>.out
 	//
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default=true
@@ -292,7 +292,7 @@ type PluginConfigNcclDebug struct {
 	// If the path does not exist, it will be created by the plugin.
 	//
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:default="/opt/soperator-outputs/%h/nccl_logs"
+	// +kubebuilder:default="/opt/soperator-outputs/nccl_logs"
 	OutputDirectory string `json:"outputDirectory,omitempty"`
 }
 

config/crd/bases/slurm.nebius.ai_slurmclusters.yaml

@@ -1292,7 +1292,7 @@ spec:
                   ncclDebug:
                     enabled: false
                     logLevel: INFO
-                    outputDirectory: /opt/soperator-outputs/%h/nccl_logs
+                    outputDirectory: /opt/soperator-outputs/nccl_logs
                     outputToFile: true
                     outputToStdOut: false
                     required: false
@@ -1336,7 +1336,7 @@ spec:
                     default:
                       enabled: false
                       logLevel: INFO
-                      outputDirectory: /opt/soperator-outputs/%h/nccl_logs
+                      outputDirectory: /opt/soperator-outputs/nccl_logs
                       outputToFile: true
                       outputToStdOut: false
                       required: false
@@ -1357,7 +1357,7 @@ spec:
                         - TRACE
                         type: string
                       outputDirectory:
-                        default: /opt/soperator-outputs/%h/nccl_logs
+                        default: /opt/soperator-outputs/nccl_logs
                         description: |-
                           OutputDirectory defines a directory path where OutputToFile has to be created.
 
@@ -1371,7 +1371,7 @@ spec:
                         description: |-
                           OutputToFile defines whether to additionally redirect `NCCL_DEBUG` outputs to the output file.
                           Output filename will have the following format:
-                           <JOB_ID>.<STEP_ID>.out
+                           <WORKER_NAME>.<JOB_ID>.<STEP_ID>.out
                         type: boolean
                       outputToStdOut:
                         default: false

config/manager/kustomization.yaml

@@ -3,4 +3,4 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/slurm-operator
-    newTag: 1.21.4
+    newTag: 1.21.5

config/manager/manager.yaml

@@ -87,7 +87,7 @@ spec:
             value: "false"
           - name: SLURM_OPERATOR_WATCH_NAMESPACES
             value: "*"
-        image: controller:1.21.4
+        image: controller:1.21.5
         imagePullPolicy: Always
         name: manager
         securityContext:

config/soperatorchecks/kustomization.yaml

@@ -4,7 +4,7 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/soperatorchecks
-    newTag: 1.21.4
+    newTag: 1.21.5
 patches:
   # Protect the /metrics endpoint by putting it behind auth.
   # If you want your controller-manager to expose the /metrics

docs/features.md

@@ -113,56 +113,50 @@ At the moment, the following information is gathered and can be viewed by users:
 
 ### Centralized Logging Scheme
 
-Soperator implements a centralized logging system that automatically collects and categorizes Slurm workload outputs. Logs are organized by worker node to optimize filesystem performance and processed by OpenTelemetry collectors for centralized analysis.
+Soperator implements a centralized logging system that automatically collects and categorizes Slurm workload outputs. Logs are organized by type and processed by OpenTelemetry collectors for centralized analysis.
 
 #### Directory Structure
 
-Logs are separated by worker node to prevent filesystem contention on the shared jail storage:
+Logs are organized in a flat structure by log type:
 
 ```
 /opt/soperator-outputs/
-├── worker-0/
-│   ├── nccl_logs/      # NCCL debug outputs from worker-0
-│   ├── slurm_jobs/     # Slurm job outputs from worker-0
-│   └── slurm_scripts/  # Script outputs (prolog, epilog, health checks) from worker-0
-├── worker-1/
-│   ├── nccl_logs/
-│   ├── slurm_jobs/
-│   └── slurm_scripts/
-└── ...
+├── nccl_logs/      # NCCL debug outputs from all workers
+├── slurm_jobs/     # Slurm job outputs from all workers
+└── slurm_scripts/  # Script outputs (prolog, epilog, health checks) from all workers
 ```
 
 #### Logging Schema
 
-Log files follow simplified naming patterns without worker prefixes (since worker identity is determined by directory structure):
+Log files include the worker name at the beginning of the filename for easy identification:
 
 **NCCL Logs:**
 ```
-job_id.job_step_id.out
-Example: 12345.67890.out (in /opt/soperator-outputs/worker-0/nccl_logs/)
+worker_name.job_id.job_step_id.out
+Example: worker-0.12345.67890.out
 ```
 
 **Slurm Jobs:**
 ```
-job_name.job_id[.array_id].out
+worker_name.job_name.job_id[.array_id].out
 Examples:
-- benchmark.12345.out
-- training.12345.1.out (array job)
+- worker-1.benchmark.12345.out
+- worker-2.training.12345.1.out (array job)
 ```
 
 **Slurm Scripts:**
 ```
-script_name[.context].out
+worker_name.script_name.context.out
 Examples:
-- health_checker.prolog.out
-- cleanup_enroot.epilog.out
+- worker-0.health_checker.prolog.out
+- worker-3.cleanup_enroot.epilog.out
 ```
 
 #### Generated Labels
 
-The logging system automatically extracts metadata and creates the following labels:
+The logging system automatically extracts metadata from filenames and creates the following labels:
 
-- `worker_name`: Worker pod identifier extracted from directory path
+- `slurm_node_name`: Slurm worker node identifier extracted from filename (e.g., "worker-0", "worker-1")
 - `log_type`: Category (nccl_logs, slurm_jobs, slurm_scripts)
 - `job_id`, `job_step_id`: For NCCL logs
 - `job_name`, `job_array_id`: For Slurm job logs

fluxcd/environment/nebius-cloud/dev/bootstrap/flux-kustomization.yaml

@@ -11,7 +11,7 @@ spec:
     name: nebius-cloud
   postBuild:
     substitute:
-      soperator_version: 1.21.4
+      soperator_version: 1.21.5
   path: "./fluxcd/enviroment/nebius-cloud/dev"
   prune: true
   timeout: 1m

fluxcd/environment/nebius-cloud/prod/bootstrap/flux-kustomization.yaml

@@ -10,7 +10,7 @@ spec:
     name: nebius-cloud
   postBuild:
     substitute:
-      soperator_version: 1.21.4
+      soperator_version: 1.21.5
   path: "./fluxcd/enviroment/nebius-cloud/prod"
   prune: false
   timeout: 1m

helm/nodeconfigurator/Chart.yaml

@@ -13,9 +13,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.21.4
+version: 1.21.5
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.21.4"
+appVersion: "1.21.5"