values.yaml

clusterName: "slurm1"
# Additional annotations for the cluster
annotations: {}
# Add appArmor profile to the cluster
useDefaultAppArmorProfile: false
# Maintenance defines the maintenance window for the cluster.
# It can have the following values:
# - none: No maintenance is performed. The cluster operates normally.
# - downscale: Scales down all components to 0.
# - downscaleAndDeletePopulateJail: Scales down all components to 0 and deletes the kubernetes Kind Jobs populateJail.
# - downscaleAndOverwritePopulateJail: Scales down all components to 0 and overwrite populateJail (same as overwrite=true).
# - skipPopulateJail: Skips the execution of the populateJail job during maintenance.
maintenance: "none"
# Slurm cluster type. Can be now gpu or cpu
clusterType: gpu
# CUDA version used for populate-jail image selection.
cudaVersion: 12.9.0
# partitionConfiguration define partition configuration of slurm worker nodes
# https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION
partitionConfiguration:
  # ConfigType defines what behaviour should be used for partition generation.
  # - default: generates 3 standard partitions (main, hidden, background)
  # - custom: uses raw partition configuration strings from rawConfig
  # - structured: enables NodeSet references and structured partition definitions
  configType: "default"
  # RawConfig defines partition configuration as list of string started with PartitionName.
  # It is used when configType == "custom".
  # Example for custom ConfigType:
  rawConfig: []
  #    - PartitionName=low_priority Nodes=low_priority Default=YES MaxTime=INFINITE State=UP PriorityTier=1
  #    - PartitionName=high_priority  Nodes=high_priority Default=NO MaxTime=INFINITE State=UP PriorityTier=2
  # If Nodes present, they must not contain node names: use only nodeset values, "ALL" or "".
  # Specifying specific nodes is not supported since Dynamic Nodes are used.
  # For more details, see https://slurm.schedmd.com/dynamic_nodes.html#partitions.

  # Partitions define partition configuration as a list of structured Partition[s].
  # It is used when configType == "structured".
  partitions: []
  # Example partitions (similar to default partitions):
  #  - name: "main"
  #    isAll: true
  #    config: "Default=YES MaxTime=INFINITE State=UP PriorityTier=10"
  #  - name: "hidden"
  #    isAll: true
  #    config: "Hidden=YES MaxTime=INFINITE State=UP PriorityTier=10"
  #  - name: "background"
  #    isAll: true
  #    config: "Hidden=YES MaxTime=INFINITE State=UP PriorityTier=1"
  # Example with NodeSet references:
  #  - name: "gpu-partition"
  #    nodeSetRefs:
  #      - "gpu-nodesets"
  #    config: "Default=YES MaxTime=8:00:00 State=UP PriorityTier=10"
  #  - name: "cpu-partition"
  #    nodeSetRefs:
  #      - "cpu-nodesets"
  #    config: "Default=NO MaxTime=INFINITE State=UP PriorityTier=5"
# List of features to be enabled on worker nodes. Each feature object has:
# - name: (Required) The name of the feature.
# - hostlist_expr: (Required) A Slurm hostlist expression, e.g. "workers-[0-2,10],workers-[3-5]".
#   Soperator will run these workers with the feature name.
# - nodeset_name: (Optional) The Slurm nodeset name to be provisioned using this feature.
#   This nodeset may be used in conjunction with partitions.

# HealthCheckConfig defines Slurm health check configuration.
healthCheckConfig: null
# K8s node filters used in Slurm node specifications. Define which nodes should be used to schedule pods to
k8sNodeFilters:
  - name: gpu
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: "nebius.com/node-group-id"
                  operator: In
                  values:
                    - "node-group-id-here"
    tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
  - name: no-gpu
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: "nebius.com/node-group-id"
                  operator: In
                  values:
                    - "node-group-id-here"
# Sources for the volumes used in Slurm node specifications
# You can use existing pvc or create via helm if `createPVC: true`
volumeSources:
  - name: controller-spool
    createPVC: false
    storageClassName: ""
    size: ""
    persistentVolumeClaim:
      claimName: "controller-spool-pvc"
      readOnly: false
  - name: jail
    createPVC: false
    storageClassName: ""
    size: ""
    persistentVolumeClaim:
      claimName: "jail-pvc"
      readOnly: false
#  - name: jail-snapshot
#    createPVC: false
#    storageClassName: ""
#    size: ""
#    persistentVolumeClaim:
#      claimName: "jail-snapshot-pvc"
#      readOnly: true
#  - name: mlperf-sd
#    createPVC: false
#    storageClassName: ""
#    size: ""
#    persistentVolumeClaim:
#      claimName: "jail-submount-mlperf-sd-pvc"
#      readOnly: false

# Secret references needed for Slurm cluster operation
secrets: {}
# Secret reference required for login sshd. If secret name empty - operator generate own secret with keys
# sshdKeysName: ""
# Job performing initial jail file system population
populateJail:
  imagePullPolicy: "IfNotPresent"
  appArmorProfile: "unconfined"
  # Name of the k8s node filter
  k8sNodeFilterName: "gpu"
  # Controls if the pod containers can use the host user namespace
  # hostUsers: false
  # Configuration of the volume containing the custom initial jail content (the default content is used if not set)
  jailSnapshotVolume: null
  #  jailSnapshotVolume:
  #    volumeSourceName: "jail-snapshot"
  overwrite: false
slurmConfig:
  defMemPerNode: 0
  defCpuPerGPU: 4
  completeWait: 5
  prolog: /opt/slurm_scripts/prolog.sh
  epilog: /opt/slurm_scripts/epilog.sh
  taskPluginParam: ""
  maxJobCount: 20000
  minJobAge: 28800
  messageTimeout: 60
  topologyPlugin: "topology/tree"
  topologyParam: "SwitchAsNodeRank"
customSlurmConfig: ""
customCgroupConfig: ""
mpiConfig:
  pmixEnv: "OMPI_MCA_btl_tcp_if_include=eth0"
plugStackConfig:
  pyxis:
    required: true
    importerPath: /opt/slurm_scripts/pyxis_caching_importer.sh
  ncclDebug:
    required: false
    enabled: true
    logLevel: "INFO"
    outputToFile: true
    outputToStdOut: false
    outputDirectory: "/opt/soperator-outputs/nccl_logs"
  customPlugins: []
#  customPlugins:
#    - required: true
#      path: custom.so
#      arguments:
#        foo: bar

slurmNodes:
  sssd:
    enabled: false
    sssdDebugLevel: 0
    imagePullPolicy: "IfNotPresent"
    appArmorProfile: "unconfined"
    command: []
    args: []
    resources:
      cpu: "200m"
      memory: "500Mi"
      ephemeralStorage: "5Gi"
    sssdConfSecretRefName: ""
    sssdLdapCAConfigMapRefName: ""
  accounting:
    enabled: false
    k8sNodeFilterName: "no-gpu"
    # Controls if the pod containers can use the host user namespace
    # hostUsers: false
    slurmConfig:
      accountingStorageTRES: "CPU,Mem,Node,VMem,Gres/gpu"
      jobAcctGatherType: "jobacct_gather/cgroup"
      jobAcctGatherFrequency: 0
      priorityWeightAge: 0
      priorityWeightFairshare: 0
      priorityWeightQOS: 0
    slurmdbdConfig:
      archiveEvents: "yes"
      archiveJobs: "yes"
      archiveResvs: "yes"
      archiveSteps: "no"
      archiveSuspend: "no"
      archiveTXN: "no"
      archiveUsage: "yes"
      debugLevel: "info"
      tcpTimeout: 2
      purgeEventAfter: "1month"
      purgeJobAfter: "12month"
      purgeResvAfter: "1month"
      purgeStepAfter: "1month"
      purgeSuspendAfter: "1month"
      purgeTXNAfter: "12month"
      purgeUsageAfter: "24month"
    slurmdbd:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      port: 6819
      resources:
        cpu: "1000m"
        memory: "3Gi"
        ephemeralStorage: "10Gi"
    munge:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      resources:
        cpu: "1000m"
        memory: "1Gi"
        ephemeralStorage: "5Gi"
    customInitContainers: []
    externalDB:
      enabled: false
      # host: ""
      port: 3306
      # user: ""
      # passwordSecretKeyRef:
      #   name: ""
      #   key: ""
      # tls:
      #   serverCASecretRef: ""
      #   clientCertSecretRef: ""
      # storageParameters:
      #   SSL_CIPHER: ""
    mariadbOperator:
      enabled: false
      protectedSecret: false
      resources:
        cpu: "1000m"
        memory: "1Gi"
        ephemeralStorage: "5Gi"
      replicas: 1
      replication: {}
      # enabled: false
      # primary: {}
      # probesEnabled: false
      # replica: {}
      # syncBinlog: false
      storage: {}
      # ephemeral: false
      # resizeInUseVolumes: false
      # size: {}
      # storageClassName: ""
      # volumeClaimTemplate: ""
      # waitForVolumeResize: false
      podSecurityContext: {}
      # fsGroup: 2000
      # runAsUser: 1000
      # runAsGroup: 3000
      # runAsNonRoot: true
      # supplementalGroups: [1000]
      securityContext: {}
      # runAsUser: 1000
      # runAsGroup: 3000
      # runAsNonRoot: true
      # capabilities:
      #   add: ["NET_ADMIN", "SYS_TIME"]
      metrics:
        enabled: true
        # username: ""
        # serviceMonitor: {}
        # jobLabel: ""
        # interval: ""
        # scrapeTimeout: ""
        # exporter: {}
        # passwordSecretKeyRef: {}
        # name: ""
        # key: ""
      priorityClassName: ""
      # affinity: {}
      # args: []
      # env: []
      # envFrom: []
      # image: ""
      # imagePullPolicy: ""
      # imagePullSecrets: []
      # initContainers: []
      # livenessProbe: {}
      # nodeSelector: {}
      # podMetadata: {}
      # securityContext: {}
      # serviceAccountName: ""
      # sidecarContainers: []
      # tolerations: []
      # resources: {}
      # volumeMounts: []
      # volumes: []
      # resources: {}
      #   limits:
      #     cpu: "100m"
      #     memory: "128Mi"
      #   requests:
      #     cpu: "50m"
      #     memory: "64Mi"
  controller:
    k8sNodeFilterName: "no-gpu"
    priorityClass: ""
    # Controls if the pod containers can use the host user namespace
    # hostUsers: false
    # RBAC settings for slurm-controller (ephemeral node power state management)
    # The power-manager binary runs inside slurmctld pod and is called by Slurm's ResumeProgram/SuspendProgram
    rbac:
      # Specifies whether RBAC resources should be created
      create: true
    # ServiceAccount settings for slurm-controller (mounted in slurmctld pod)
    serviceAccount:
      # Specifies whether a ServiceAccount should be created
      create: true
      # The name of the ServiceAccount to use.
      # If not set and create is true, a name is generated using the cluster name: <clusterName>-slurm-controller
      name: ""
    slurmctld:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      port: 6817
      resources:
        cpu: "1000m"
        memory: "3Gi"
        ephemeralStorage: "20Gi"
    munge:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      resources:
        cpu: "1000m"
        memory: "1Gi"
        ephemeralStorage: "5Gi"
    customInitContainers: []
    volumes:
      spool:
        volumeClaimTemplateSpec:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 50Gi
          storageClassName: "compute-csi-network-ssd-ext4"
      jail:
        volumeSourceName: "jail"
      customMounts: []
  login:
    size: 2
    k8sNodeFilterName: "no-gpu"
    # Controls if the pod containers can use the host user namespace
    # hostUsers: false
    sshd:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      port: 22
      resources:
        cpu: "3000m"
        memory: "9Gi"
        ephemeralStorage: "30Gi"
      command: []
      args: []
    # Authorized keys required for SSH connection to Slurm login nodes
    sshRootPublicKeys:
      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKzxkjzPQ4EyZSjan4MLGFSA18idpZicoKW7Hfff username1"
      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICL8scMKnwu+Y9S6XDACacZ54+qu+YRo2y4Ieddd username2"
    # Either `LoadBalancer` or `NodePort`
    sshdServiceType: "LoadBalancer"
    # Could be needed in case of sshdServiceType == `LoadBalancer`
    # sshdServiceLoadBalancerIP: "192.168.0.1"
    sshdServiceLoadBalancerIP: ""
    # Source IP ranges allowed to access LoadBalancer (CIDR format)
    # sshdServiceLoadBalancerSourceRanges: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
    sshdServiceLoadBalancerSourceRanges: []
    # Required in case of sshdServiceType == `NodePort`
    sshdServiceNodePort: 30022
    # Annotations to be configure on login service
    sshdServiceAnnotations: {}
    munge:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      resources:
        cpu: "500m"
        memory: "500Mi"
        ephemeralStorage: "5Gi"
    customInitContainers: []
    volumes:
      jail:
        volumeSourceName: "jail"
      jailSubMounts: []
      # jailSubMounts:
      #   - name: "mlcommons-sd-bench-data"
      #     mountPath: "/mlperf-sd"
      #     volumeSourceName: "mlperf-sd"
      customMounts: []
  exporter:
    enabled: true
    size: 1
    k8sNodeFilterName: "no-gpu"
    # Controls if the pod containers can use the host user namespace
    # hostUsers: false
    podMonitorConfig:
      jobLabel: "slurm-exporter"
      interval: "30s"
      scrapeTimeout: "28s"
    # RBAC settings for exporter
    rbac:
      # Specifies whether RBAC resources (Role/RoleBinding) should be created
      create: true
    # ServiceAccount settings for exporter
    serviceAccount:
      # Specifies whether a ServiceAccount should be created
      create: true
      # The name of the ServiceAccount to use.
      # - If empty and create is true, a ServiceAccount named "slurm-exporter-sa" will be created and used.
      # - If empty and create is false, the "default" ServiceAccount will be used and you must ensure it has the required RBAC.
      name: ""
    collectionInterval: "30s"
    exporter:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      resources:
        cpu: "250m"
        memory: "256Mi"
        ephemeralStorage: "500Mi"
    munge:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      resources:
        cpu: "1000m"
        memory: "1Gi"
        ephemeralStorage: "5Gi"
    customInitContainers: []
    volumes:
      jail:
        volumeSourceName: "jail"
  rest:
    enabled: false
    size: 2
    k8sNodeFilterName: "no-gpu"
    # Controls if the pod containers can use the host user namespace
    # hostUsers: false
    threadCount: 3
    maxConnections: 10
    rest:
      imagePullPolicy: "IfNotPresent"
      appArmorProfile: "unconfined"
      command: []
      args: []
      resources:
        cpu: "2000m"
        memory: "8Gi"
        ephemeralStorage: "500Mi"
    customInitContainers: []
sConfigController:
  # Controls if the pod containers can use the host user namespace
  # hostUsers: false
  node:
    k8sNodeFilterName: "system"
    size: 1
  container:
    imagePullPolicy: "IfNotPresent"
    resources:
      cpu: "250m"
      memory: "256Mi"
      ephemeralStorage: "500Mi"
  # These should match UID and GID in sconfigcontroller image
  runAsUid: 1001
  runAsGid: 1001
  reconfigurePollInterval: "20s"
  reconfigureWaitTimeout: "5m"
  serviceMonitor:
    enabled: true
    jobLabel: "sconfigcontroller"
    interval: "30s"
    scrapeTimeout: "28s"
  # RBAC settings for sconfigcontroller
  rbac:
    # Specifies whether RBAC resources should be created
    create: true
  # ServiceAccount settings for sconfigcontroller
  serviceAccount:
    # Specifies whether a ServiceAccount should be created
    create: true
    # The name of the ServiceAccount to use.
    # If not set and create is true, a name is generated using the cluster name: <clusterName>-sconfigcontroller
    # This field is required in the SlurmCluster CR - it will be automatically populated from this value
    name: ""
images:
  slurmctld: "cr.eu-north1.nebius.cloud/soperator/controller_slurmctld:3.0.2-slurm25.11.3"
  slurmrestd: "cr.eu-north1.nebius.cloud/soperator/slurmrestd:3.0.2-slurm25.11.3"
  slurmd: "cr.eu-north1.nebius.cloud/soperator/worker_slurmd:3.0.2-slurm25.11.3"
  sshd: "cr.eu-north1.nebius.cloud/soperator/login_sshd:3.0.2-slurm25.11.3"
  munge: "cr.eu-north1.nebius.cloud/soperator/munge:3.0.2-slurm25.11.3"
  sssd: "cr.eu-north1.nebius.cloud/soperator/sssd:2.9.4-1-63"
  populateJail: ""
  populateJailRepository: "cr.eu-north1.nebius.cloud/soperator/populate_jail"
  populateJailTag: "3.0.2-slurm25.11.3"
  slurmdbd: "cr.eu-north1.nebius.cloud/soperator/controller_slurmdbd:3.0.2-slurm25.11.3"
  soperatorExporter: "cr.eu-north1.nebius.cloud/soperator/soperator-exporter:3.0.2-slurm25.11.3"
  sConfigController: "cr.eu-north1.nebius.cloud/soperator/sconfigcontroller:3.0.2"
  mariaDB: "docker-registry1.mariadb.com/library/mariadb:12.1.2"
# Configuration for slurm scripts.
slurmScripts:
  # Base scripts content can be overridden here
  check_runner.py: null
  prolog.sh: null
  epilog.sh: null
  hc_program.sh: null
  pyxis_caching_importer.sh: null
  # Built-in scripts can be overridden or disabled here
  # Keys must match file names in slurm_scripts/
  builtIn:
    alloc_gpus_busy.drain.sh:
      enabled: true
      customContent: null
      customConfig: null
    alloc_gpus_busy.undrain.sh:
      enabled: true
      customContent: null
      customConfig: null
    alloc_mem_used.drain.sh:
      enabled: true
      customContent: null
      customConfig: null
    alloc_mem_used.undrain.sh:
      enabled: true
      customContent: null
      customConfig: null
    boot_disk_full.sh:
      enabled: true
      customContent: null
      customConfig: null
    chmod_enroot_layers.sh:
      enabled: true
      customContent: null
      customConfig: null
    cleanup_enroot.sh:
      enabled: true
      customContent: null
      customConfig: null
    cleanup_scratch_data.sh:
      enabled: true
      customContent: null
      customConfig: null
    drop_page_cache.sh:
      enabled: true
      customContent: null
      customConfig: null
    drop_posix_shmem.sh:
      enabled: true
      customContent: null
      customConfig: null
    gpu_health_check.py:
      enabled: true
      customContent: null
      customConfig: null
    job_tmpfs_delete.sh:
      enabled: true
      customContent: null
      customConfig: null
    job_tmpfs_delete_leftover.sh:
      enabled: true
      customContent: null
      customConfig: null
    job_tmpfs_recreate.sh:
      enabled: true
      customContent: null
      customConfig: null
    map_job_dcgm.sh:
      enabled: true
      customContent: null
      customConfig: null
    nvme_raid_health.sh:
      enabled: false
      customContent: null
      customConfig: null
    unmap_job_dcgm.sh:
      enabled: true
      customContent: null
      customConfig: null
  extra: null
  # extra:
  #   example.sh:
  #     config: |-
  #     {
  #       "name": "exmaple",
  #       "command": "./example.sh",
  #       "platforms": ["any"],
  #       "skip_for_cpu_jobs": false,
  #       "skip_for_partial_gpu_jobs": false,
  #       "skip_for_reservation_prefixes": [],
  #       "contexts": ["prolog"],
  #       "node_states": ["any"],
  #       "on_fail": "drain",
  #       "on_ok": "undrain",
  #       "reason_base": "[some tag] $name",
  #       "reason_append_details": true,
  #       "run_in_jail": false,
  #       "log": "slurm_scripts/$worker.$name.$context.out",
  #       "need_env": []
  #     }
  #     content: |-
  #     #!/bin/bash
  #     echo "Cluster: {{ $.Values.clusterName }}"