Skip to content

Commit 8cf530a

Browse files
koct9ikoct9i
committed
Add nvidia container runtime for CRI-O
Add configuration file for CRI-O, because configuration via environment variables is not flexible enough. Signed-off-by: Konstantin Khlebnikov <[email protected]>
1 parent eb8bf69 commit 8cf530a

File tree

13 files changed

+415
-85
lines changed

13 files changed

+415
-85
lines changed

‎pkg/components/exec_node.go‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -46,24 +46,12 @@ func NewExecNode(
4646
)
4747

4848
criConfig := ytconfig.NewCRIConfigGenerator(&spec)
49-
50-
var sidecarConfig *ConfigMapBuilder
51-
if criConfig.Service == ytv1.CRIServiceContainerd {
52-
sidecarConfig = NewConfigMapBuilder(
53-
l,
54-
ytsaurus.APIProxy(),
55-
l.GetSidecarConfigMapName(consts.JobsContainerName),
56-
ytsaurus.GetResource().Spec.ConfigOverrides,
57-
)
58-
59-
sidecarConfig.AddGenerator(
60-
consts.ContainerdConfigFileName,
61-
ConfigFormatToml,
62-
func() ([]byte, error) {
63-
return criConfig.GetContainerdConfig()
64-
},
65-
)
66-
}
49+
sidecarConfig := NewJobsSidecarConfig(
50+
l,
51+
ytsaurus.APIProxy(),
52+
criConfig,
53+
ytsaurus.GetCommonSpec().ConfigOverrides,
54+
)
6755

6856
if criConfig.MonitoringPort != 0 {
6957
srv.addMonitoringPort(corev1.ServicePort{

‎pkg/components/exec_node_base.go‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ import (
1010

1111
ytv1 "github.com/ytsaurus/ytsaurus-k8s-operator/api/v1"
1212

13+
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/apiproxy"
1314
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/consts"
15+
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/labeller"
1416
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/resources"
1517
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/ytconfig"
1618
)
@@ -110,7 +112,11 @@ func (n *baseExecNode) addCRIServiceConfig(cri *ytconfig.CRIConfigGenerator, con
110112
case ytv1.CRIServiceNone:
111113
return
112114
case ytv1.CRIServiceCRIO:
113-
container.Env = append(container.Env, cri.GetCRIOEnv()...)
115+
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
116+
Name: consts.CRIOConfigVolumeName,
117+
MountPath: consts.CRIOConfigMountPoint,
118+
ReadOnly: true,
119+
})
114120
case ytv1.CRIServiceContainerd:
115121
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
116122
Name: consts.ContainerdConfigVolumeName,
@@ -151,9 +157,13 @@ func (n *baseExecNode) addCRIServiceSidecar(cri *ytconfig.CRIConfigGenerator, po
151157
},
152158
}
153159

154-
if cri.Service == ytv1.CRIServiceContainerd {
160+
switch cri.Service {
161+
case ytv1.CRIServiceContainerd:
155162
configPath := path.Join(consts.ContainerdConfigMountPoint, consts.ContainerdConfigFileName)
156163
container.Args = []string{"--config", configPath}
164+
case ytv1.CRIServiceCRIO:
165+
configPath := path.Join(consts.CRIOConfigMountPoint, consts.CRIOConfigFileName)
166+
container.Args = []string{"--config", configPath, "--config-dir", ""}
157167
}
158168

159169
n.addCRIServiceConfig(cri, &container)
@@ -204,3 +214,40 @@ func (n *baseExecNode) sidecarConfigNeedsReload() bool {
204214
}
205215
return needsReload
206216
}
217+
218+
func NewJobsSidecarConfig(
219+
labeller *labeller.Labeller,
220+
apiProxy apiproxy.APIProxy,
221+
criConfig *ytconfig.CRIConfigGenerator,
222+
configOverrides *corev1.LocalObjectReference,
223+
) *ConfigMapBuilder {
224+
config := NewConfigMapBuilder(
225+
labeller,
226+
apiProxy,
227+
labeller.GetSidecarConfigMapName(consts.JobsContainerName),
228+
configOverrides,
229+
)
230+
231+
switch criConfig.Service {
232+
case ytv1.CRIServiceNone:
233+
config = nil
234+
case ytv1.CRIServiceContainerd:
235+
config.AddGenerator(
236+
consts.ContainerdConfigFileName,
237+
ConfigFormatToml,
238+
func() ([]byte, error) {
239+
return criConfig.GetContainerdConfig()
240+
},
241+
)
242+
case ytv1.CRIServiceCRIO:
243+
config.AddGenerator(
244+
consts.CRIOConfigFileName,
245+
ConfigFormatToml,
246+
func() ([]byte, error) {
247+
return criConfig.GetCRIOConfig()
248+
},
249+
)
250+
}
251+
252+
return config
253+
}

‎pkg/components/exec_node_remote.go‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
corev1 "k8s.io/api/core/v1"
7+
"k8s.io/apimachinery/pkg/util/intstr"
78

89
ytv1 "github.com/ytsaurus/ytsaurus-k8s-operator/api/v1"
910
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/apiproxy"
@@ -44,23 +45,20 @@ func NewRemoteExecNodes(
4445
)
4546

4647
criConfig := ytconfig.NewCRIConfigGenerator(&spec)
48+
sidecarConfig := NewJobsSidecarConfig(
49+
l,
50+
proxy,
51+
criConfig,
52+
commonSpec.ConfigOverrides,
53+
)
4754

48-
var sidecarConfig *ConfigMapBuilder
49-
if criConfig.Service == ytv1.CRIServiceContainerd {
50-
sidecarConfig = NewConfigMapBuilder(
51-
l,
52-
proxy,
53-
l.GetSidecarConfigMapName(consts.JobsContainerName),
54-
commonSpec.ConfigOverrides,
55-
)
56-
57-
sidecarConfig.AddGenerator(
58-
consts.ContainerdConfigFileName,
59-
ConfigFormatToml,
60-
func() ([]byte, error) {
61-
return criConfig.GetContainerdConfig()
62-
},
63-
)
55+
if criConfig.MonitoringPort != 0 {
56+
srv.addMonitoringPort(corev1.ServicePort{
57+
Name: consts.CRIServiceMonitoringPortName,
58+
Protocol: corev1.ProtocolTCP,
59+
Port: criConfig.MonitoringPort,
60+
TargetPort: intstr.FromInt32(criConfig.MonitoringPort),
61+
})
6462
}
6563

6664
return &RemoteExecNode{

‎pkg/consts/cmd.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ const (
5050

5151
CRIServiceSocketName = "cri.sock"
5252

53+
CRIOConfigVolumeName = "config-crio"
54+
CRIOConfigMountPoint = "/config/crio"
55+
CRIOConfigFileName = "crio.conf"
56+
5357
CRINamespace = "yt"
5458
CRIBaseCgroup = "/yt"
5559

‎pkg/ytconfig/cri.go‎

Lines changed: 94 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,24 @@ import (
1212
"github.com/ytsaurus/ytsaurus-k8s-operator/pkg/consts"
1313
)
1414

15+
const (
16+
runtimeTypeOCI = "oci"
17+
18+
runtimeNameRunc = "runc"
19+
crioRuntimePathRunc = "/usr/libexec/crio/runc"
20+
21+
runtimeNameCrun = "crun"
22+
crioRuntimePathCrun = "/usr/libexec/crio/crun"
23+
24+
runtimeNameNvidia = "nvidia"
25+
runtimePathNvidia = "/usr/bin/nvidia-container-runtime"
26+
27+
crioMonitorCgroup = "pod"
28+
crioMonitorPath = "/usr/libexec/crio/conmon"
29+
30+
shmSizeAnnotation = "io.kubernetes.cri-o.ShmSize"
31+
)
32+
1533
type CRIConfigGenerator struct {
1634
Service ytv1.CRIServiceType
1735
Spec ytv1.CRIJobEnvironmentSpec
@@ -69,29 +87,84 @@ func (cri *CRIConfigGenerator) GetCRIToolsEnv() []corev1.EnvVar {
6987
return env
7088
}
7189

72-
func (cri *CRIConfigGenerator) GetCRIOEnv() []corev1.EnvVar {
73-
var env []corev1.EnvVar
90+
func (cri *CRIConfigGenerator) GetCRIOConfig() ([]byte, error) {
91+
// See https://github.com/cri-o/cri-o/blob/main/docs/crio.conf.5.md
92+
93+
crioAPI := map[string]any{
94+
"listen": cri.GetSocketPath(),
95+
}
96+
97+
crioImage := map[string]any{}
98+
99+
crioMetrics := map[string]any{}
100+
101+
crioRuntimeRuntimes := map[string]any{
102+
runtimeNameRunc: map[string]any{
103+
"runtime_type": runtimeTypeOCI,
104+
"runtime_path": crioRuntimePathRunc,
105+
"allowed_annotations": []string{
106+
shmSizeAnnotation,
107+
},
108+
"monitor_cgroup": crioMonitorCgroup,
109+
"monitor_path": crioMonitorPath,
110+
},
111+
runtimeNameCrun: map[string]any{
112+
"runtime_type": runtimeTypeOCI,
113+
"runtime_path": crioRuntimePathCrun,
114+
"allowed_annotations": []string{
115+
shmSizeAnnotation,
116+
},
117+
"monitor_cgroup": crioMonitorCgroup,
118+
"monitor_path": crioMonitorPath,
119+
},
120+
}
121+
122+
crioRuntime := map[string]any{
123+
"cgroup_manager": "cgroupfs",
124+
"conmon_cgroup": crioMonitorCgroup,
125+
"default_runtime": runtimeNameCrun,
126+
"runtimes": crioRuntimeRuntimes,
127+
}
128+
129+
crio := map[string]any{
130+
"api": crioAPI,
131+
"image": crioImage,
132+
"metrics": crioMetrics,
133+
"runtime": crioRuntime,
134+
}
135+
136+
config := map[string]any{
137+
"crio": crio,
138+
}
74139

75-
// See https://github.com/cri-o/cri-o/blob/main/docs/crio.8.md
76-
env = append(env,
77-
corev1.EnvVar{Name: "CONTAINER_LISTEN", Value: cri.GetSocketPath()},
78-
corev1.EnvVar{Name: "CONTAINER_CGROUP_MANAGER", Value: "cgroupfs"},
79-
corev1.EnvVar{Name: "CONTAINER_CONMON_CGROUP", Value: "pod"},
80-
)
81140
if cri.StoragePath != nil {
82-
env = append(env, corev1.EnvVar{Name: "CONTAINER_ROOT", Value: *cri.StoragePath})
141+
crio["root"] = *cri.StoragePath
83142
}
143+
84144
if cri.Spec.SandboxImage != nil {
85-
env = append(env, corev1.EnvVar{Name: "CONTAINER_PAUSE_IMAGE", Value: *cri.Spec.SandboxImage})
145+
crioImage["pause_image"] = *cri.Spec.SandboxImage
86146
}
147+
87148
if cri.MonitoringPort != 0 {
88-
env = append(env,
89-
corev1.EnvVar{Name: "CONTAINER_ENABLE_METRICS", Value: "true"},
90-
corev1.EnvVar{Name: "CONTAINER_METRICS_HOST", Value: ""},
91-
corev1.EnvVar{Name: "CONTAINER_METRICS_PORT", Value: fmt.Sprintf("%d", cri.MonitoringPort)},
92-
)
149+
crioMetrics["enable_metrics"] = true
150+
crioMetrics["metrics_host"] = ""
151+
crioMetrics["metrics_port"] = cri.MonitoringPort
93152
}
94-
return env
153+
154+
if cri.Runtime != nil && cri.Runtime.Nvidia != nil {
155+
crioRuntimeRuntimes[runtimeNameNvidia] = map[string]any{
156+
"runtime_type": runtimeTypeOCI,
157+
"runtime_path": runtimePathNvidia,
158+
"allowed_annotations": []string{
159+
shmSizeAnnotation,
160+
},
161+
"monitor_cgroup": crioMonitorCgroup,
162+
"monitor_path": crioMonitorPath,
163+
}
164+
crioRuntime["default_runtime"] = runtimeNameNvidia
165+
}
166+
167+
return marshallYsonConfig(config)
95168
}
96169

97170
func (cri *CRIConfigGenerator) GetContainerdConfig() ([]byte, error) {
@@ -143,25 +216,25 @@ func (cri *CRIConfigGenerator) GetContainerdConfig() ([]byte, error) {
143216

144217
func (cri *CRIConfigGenerator) getContainerdRuntimes() (runtimes map[string]any, defaultRuntimeName string) {
145218
runtimes = map[string]any{
146-
"runc": map[string]any{
219+
runtimeNameRunc: map[string]any{
147220
"runtime_type": "io.containerd.runc.v2",
148221
"sandbox_mode": "podsandbox",
149222
"options": map[string]any{
150223
"SystemdCgroup": false,
151224
},
152225
},
153226
}
154-
defaultRuntimeName = "runc"
227+
defaultRuntimeName = runtimeNameRunc
155228

156229
if cri.Runtime != nil && cri.Runtime.Nvidia != nil {
157-
runtimes["nvidia"] = map[string]any{
230+
runtimes[runtimeNameNvidia] = map[string]any{
158231
"runtime_type": "io.containerd.runc.v2",
159232
"sandbox_mode": "podsandbox",
160233
"options": map[string]any{
161-
"BinaryName": "/usr/bin/nvidia-container-runtime",
234+
"BinaryName": runtimePathNvidia,
162235
},
163236
}
164-
defaultRuntimeName = "nvidia"
237+
defaultRuntimeName = runtimeNameNvidia
165238
}
166239

167240
return runtimes, defaultRuntimeName

‎test/r8r/canondata/Components reconciler With CRI and NVIDIA container runtime - CRI-O Test/ConfigMap yt-exec-node-jobs-config.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
apiVersion: v1
2+
data:
3+
crio.conf: |
4+
[crio]
5+
root = "/yt/node-data/image-cache"
6+
[crio.api]
7+
listen = "/yt/node-data/image-cache/cri.sock"
8+
[crio.image]
9+
[crio.metrics]
10+
enable_metrics = true
11+
metrics_host = ""
12+
metrics_port = 10026
13+
[crio.runtime]
14+
cgroup_manager = "cgroupfs"
15+
conmon_cgroup = "pod"
16+
default_runtime = "nvidia"
17+
[crio.runtime.runtimes]
18+
[crio.runtime.runtimes.crun]
19+
allowed_annotations = ["io.kubernetes.cri-o.ShmSize"]
20+
monitor_cgroup = "pod"
21+
monitor_path = "/usr/libexec/crio/conmon"
22+
runtime_path = "/usr/libexec/crio/crun"
23+
runtime_type = "oci"
24+
[crio.runtime.runtimes.nvidia]
25+
allowed_annotations = ["io.kubernetes.cri-o.ShmSize"]
26+
monitor_cgroup = "pod"
27+
monitor_path = "/usr/libexec/crio/conmon"
28+
runtime_path = "/usr/bin/nvidia-container-runtime"
29+
runtime_type = "oci"
30+
[crio.runtime.runtimes.runc]
31+
allowed_annotations = ["io.kubernetes.cri-o.ShmSize"]
32+
monitor_cgroup = "pod"
33+
monitor_path = "/usr/libexec/crio/conmon"
34+
runtime_path = "/usr/libexec/crio/runc"
35+
runtime_type = "oci"
36+
kind: ConfigMap
37+
metadata:
38+
creationTimestamp: null
39+
generation: 1
40+
labels:
41+
app.kubernetes.io/component: yt-exec-node
42+
app.kubernetes.io/managed-by: ytsaurus-k8s-operator
43+
app.kubernetes.io/name: yt-exec-node
44+
app.kubernetes.io/part-of: yt-test-ytsaurus
45+
yt_component: test-ytsaurus-yt-exec-node
46+
ytsaurus.tech/cluster-name: test-ytsaurus
47+
name: yt-exec-node-jobs-config
48+
namespace: ytsaurus-components
49+
ownerReferences:
50+
- apiVersion: cluster.ytsaurus.tech/v1
51+
blockOwnerDeletion: true
52+
controller: true
53+
kind: Ytsaurus
54+
name: test-ytsaurus
55+
uid: ""
56+
resourceVersion: "1"

0 commit comments

Comments
 (0)