k8s-launch-kit/l8k-config.yaml at main · NVIDIA/k8s-launch-kit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
networkOperator:
  version: v26.1.0
  componentVersion: network-operator-v26.1.0
  repository: nvcr.io/nvidia/mellanox
  namespace: nvidia-network-operator

podNamespace: default

docaDriver:
  enable: true
  version: doca3.3.0-26.01-1.0.0.0-0
  unloadStorageModules: true
  enableNFSRDMA: false
  unloadDependentModules: true

nvIpam:
  poolName: nv-ipam-pool
  # Option 1: Auto-generate subnets (used when subnets list is empty).
  # Each group gets its own unique subnet slice; gateway = network address + 1.
  startingSubnet: "192.168.0.0"
  mask: 22
  offset: 1
  # Option 2: Manually list subnets (takes precedence if non-empty)
  # subnets:
  # - subnet: 192.168.2.0/24
  #   gateway: 192.168.2.1
  # - subnet: 192.168.3.0/24
  #   gateway: 192.168.3.1
  # - subnet: 192.168.4.0/24
  #   gateway: 192.168.4.1
  # - subnet: 192.168.5.0/24
  #   gateway: 192.168.5.1
  # - subnet: 192.168.6.0/24
  #   gateway: 192.168.6.1
  # - subnet: 192.168.7.0/24
  #   gateway: 192.168.7.1
  # - subnet: 192.168.8.0/24
  #   gateway: 192.168.8.1
  # - subnet: 192.168.9.0/24
  #   gateway: 192.168.9.1
  # - subnet: 192.168.10.0/24
  #   gateway: 192.168.10.1

sriov:
  ethernetMtu: 9000
  infinibandMtu: 4000
  numVfs: 8
  priority: 90
  resourceName: sriov_resource
  networkName: sriov-network

hostdev:
  resourceName: hostdev_resource
  networkName: hostdev-network

rdmaShared:
  resourceName: rdma_shared_resource # with multiple pools, _rail_0, _rail_1, etc. suffixes are added to the resource name
  hcaMax: 63

ipoib:
  networkName: ipoib-network # with multiple networks, -rail-0, -rail-1, etc. suffixes are added to the network name

macvlan:
  networkName: macvlan-network # with multiple networks, -rail-0, -rail-1, etc. suffixes are added to the network name

nicConfigurationOperator:
  deployNicInterfaceNameTemplate: true
  rdmaPrefix: "rdma_r%rail_id%"
  netdevPrefix: "eth_r%rail_id%"

spectrumX:
  nicType: "1023" # "1023" for ConnectX-8, "a2dc" for BlueField-3 SuperNIC
  overlay: "none"
  rdmaPrefix: "roce_p%plane_id%_r%rail_id%"
  netdevPrefix: "eth_p%plane_id%_r%rail_id%"

profile:
  fabric: ethernet # infiniband, ethernet TODO consider ETH/IB
  deployment: sriov # rdma_shared, sriov, host_device
  multirail: false
  spectrumX: # Spectrum-X configuration (set enable: true or use --spectrum-x CLI flag)
    enable: false # CLI parameter (overrides this value): --spectrum-x
    spcxVersion: "RA2.1" # CLI parameter (overrides this value): --spcx-version
    multiplaneMode: swplb # CLI parameter (overrides this value): --multiplane-mode (swplb, hwplb, uniplane)
    numberOfPlanes: 4 # CLI parameter (overrides this value): --number-of-planes, also used as pfsPerNic
  ai: false

clusterConfig:
  - identifier: ""
    capabilities:
      nodes:
        sriov: true # has nodes with feature.node.kubernetes.io/pci-15b3.present=true
        rdma: true # has nodes with feature.node.kubernetes.io/rdma.capable=true
        ib: true # has nodes with IB capable NICs (find via nic config op)
    workerNodes: ["worker-0", "worker-1", "worker-2"]
    pfs:
      - deviceID: 1023
        pciAddress: 0000:05:00.0
        rdmaDevice: "mlx5_0"
        networkInterface: "net1"
        traffic: east-west
        rail: 0
      - deviceID: 1023
        pciAddress: 0000:75:00.0
        rdmaDevice: "mlx5_1"
        networkInterface: "net2"
        traffic: east-west
        rail: 1
      - deviceID: 1023
        pciAddress: 0000:85:00.0
        rdmaDevice: "mlx5_2"
        networkInterface: "net3"
        traffic: east-west
        rail: 2
      - deviceID: 1023
        pciAddress: 0000:f5:00.0
        rdmaDevice: "mlx5_3"
        networkInterface: "net4"
        traffic: east-west
        rail: 3
      - deviceID: 1023
        pciAddress: 0000:6a:00.0
        rdmaDevice: "mlx5_4"
        networkInterface: "net5"
        traffic: north-south
    nodeSelector:
      feature.node.kubernetes.io/pci-15b3.present: "true"