Skip to content

Commit 07e2ab2

Browse files
yevgeny-shnaidmank8s-ci-robot
yevgeny-shnaidman
authored and
k8s-ci-robot
committed
Adding init container for device plugin implementation
This commit creates an init container definition for the device-plugin daemonset, based on the definitions in the Module CR. In case the ini-container is not defined in the Module CR, then it won't be defined for the device-plugin daemonset
1 parent 9254096 commit 07e2ab2

File tree

2 files changed

+194
-149
lines changed

2 files changed

+194
-149
lines changed

‎internal/controllers/device_plugin_reconciler.go‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -364,19 +364,8 @@ func (dsci *daemonSetCreatorImpl) setDevicePluginAsDesired(
364364
Finalizers: []string{constants.NodeLabelerFinalizer},
365365
},
366366
Spec: v1.PodSpec{
367-
Containers: []v1.Container{
368-
{
369-
Args: mod.Spec.DevicePlugin.Container.Args,
370-
Command: mod.Spec.DevicePlugin.Container.Command,
371-
Env: mod.Spec.DevicePlugin.Container.Env,
372-
Name: "device-plugin",
373-
Image: mod.Spec.DevicePlugin.Container.Image,
374-
ImagePullPolicy: mod.Spec.DevicePlugin.Container.ImagePullPolicy,
375-
Resources: mod.Spec.DevicePlugin.Container.Resources,
376-
SecurityContext: &v1.SecurityContext{Privileged: ptr.To(true)},
377-
VolumeMounts: append(mod.Spec.DevicePlugin.Container.VolumeMounts, containerVolumeMounts...),
378-
},
379-
},
367+
InitContainers: generatePodContainerSpec(mod.Spec.DevicePlugin.InitContainer, "device-plugin-init", nil),
368+
Containers: generatePodContainerSpec(&mod.Spec.DevicePlugin.Container, "device-plugin", containerVolumeMounts),
380369
PriorityClassName: "system-node-critical",
381370
ImagePullSecrets: getPodPullSecrets(mod.Spec.ImageRepoSecret),
382371
NodeSelector: nodeSelector,
@@ -390,6 +379,25 @@ func (dsci *daemonSetCreatorImpl) setDevicePluginAsDesired(
390379
return controllerutil.SetControllerReference(mod, ds, dsci.scheme)
391380
}
392381

382+
func generatePodContainerSpec(containerSpec *kmmv1beta1.DevicePluginContainerSpec, containerName string, presetVolumeMounts []v1.VolumeMount) []v1.Container {
383+
if containerSpec == nil {
384+
return nil
385+
}
386+
return []v1.Container{
387+
{
388+
Args: containerSpec.Args,
389+
Command: containerSpec.Command,
390+
Env: containerSpec.Env,
391+
Name: containerName,
392+
Image: containerSpec.Image,
393+
ImagePullPolicy: containerSpec.ImagePullPolicy,
394+
Resources: containerSpec.Resources,
395+
SecurityContext: &v1.SecurityContext{Privileged: ptr.To(true)},
396+
VolumeMounts: append(containerSpec.VolumeMounts, presetVolumeMounts...),
397+
},
398+
}
399+
}
400+
393401
func generateDevicePluginLabelsAndSelector(mod *kmmv1beta1.Module) (map[string]string, map[string]string) {
394402
labels := map[string]string{constants.ModuleNameLabel: mod.Name}
395403
nodeSelector := map[string]string{utils.GetKernelModuleReadyNodeLabel(mod.Namespace, mod.Name): ""}

‎internal/controllers/device_plugin_reconciler_test.go‎

Lines changed: 173 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -626,174 +626,211 @@ var _ = Describe("DevicePluginReconciler_setDevicePluginAsDesired", func() {
626626
Expect(ds.GetLabels()).Should(HaveKeyWithValue(versionLabel, "some version"))
627627
})
628628

629-
DescribeTable("should work as expected", func(moduleLoader *kmmv1beta1.ModuleLoaderSpec, expectedNodeSelector map[string]string) {
630-
const (
631-
dsName = "ds-name"
632-
serviceAccountName = "some-service-account"
633-
)
629+
DescribeTable("should work as expected",
630+
func(moduleLoader *kmmv1beta1.ModuleLoaderSpec, expectedNodeSelector map[string]string, withInitContainer bool) {
631+
const (
632+
dsName = "ds-name"
633+
serviceAccountName = "some-service-account"
634+
)
634635

635-
dpVol := v1.Volume{
636-
Name: "test-volume",
637-
VolumeSource: v1.VolumeSource{},
638-
}
636+
dpVol := v1.Volume{
637+
Name: "test-volume",
638+
VolumeSource: v1.VolumeSource{},
639+
}
639640

640-
dpVolMount := v1.VolumeMount{
641-
Name: "some-dp-volume-mount",
642-
MountPath: "/some/path",
643-
}
641+
dpVolMount := v1.VolumeMount{
642+
Name: "some-dp-volume-mount",
643+
MountPath: "/some/path",
644+
}
644645

645-
repoSecret := v1.LocalObjectReference{Name: "pull-secret-name"}
646+
repoSecret := v1.LocalObjectReference{Name: "pull-secret-name"}
646647

647-
env := []v1.EnvVar{
648-
{
649-
Name: "ENV_KEY",
650-
Value: "ENV_VALUE",
651-
},
652-
}
648+
env := []v1.EnvVar{
649+
{
650+
Name: "ENV_KEY",
651+
Value: "ENV_VALUE",
652+
},
653+
}
653654

654-
resources := v1.ResourceRequirements{
655-
Limits: map[v1.ResourceName]resource.Quantity{
656-
v1.ResourceCPU: resource.MustParse("200m"),
657-
v1.ResourceMemory: resource.MustParse("4G"),
658-
},
659-
Requests: map[v1.ResourceName]resource.Quantity{
660-
v1.ResourceCPU: resource.MustParse("100m"),
661-
v1.ResourceMemory: resource.MustParse("2G"),
662-
},
663-
}
655+
resources := v1.ResourceRequirements{
656+
Limits: map[v1.ResourceName]resource.Quantity{
657+
v1.ResourceCPU: resource.MustParse("200m"),
658+
v1.ResourceMemory: resource.MustParse("4G"),
659+
},
660+
Requests: map[v1.ResourceName]resource.Quantity{
661+
v1.ResourceCPU: resource.MustParse("100m"),
662+
v1.ResourceMemory: resource.MustParse("2G"),
663+
},
664+
}
664665

665-
args := []string{"some", "args"}
666-
command := []string{"some", "command"}
666+
args := []string{"some", "args"}
667+
command := []string{"some", "command"}
667668

668-
testToleration := v1.Toleration{
669-
Key: "test-key",
670-
Value: "test-value",
671-
Effect: v1.TaintEffectNoExecute,
672-
}
669+
testToleration := v1.Toleration{
670+
Key: "test-key",
671+
Value: "test-value",
672+
Effect: v1.TaintEffectNoExecute,
673+
}
673674

674-
const ipp = v1.PullIfNotPresent
675+
const ipp = v1.PullIfNotPresent
675676

676-
mod := kmmv1beta1.Module{
677-
TypeMeta: metav1.TypeMeta{
678-
APIVersion: kmmv1beta1.GroupVersion.String(),
679-
Kind: "Module",
680-
},
681-
ObjectMeta: metav1.ObjectMeta{
682-
Name: moduleName,
683-
Namespace: namespace,
684-
},
685-
Spec: kmmv1beta1.ModuleSpec{
686-
ModuleLoader: moduleLoader,
687-
DevicePlugin: &kmmv1beta1.DevicePluginSpec{
688-
Container: kmmv1beta1.DevicePluginContainerSpec{
689-
Args: args,
690-
Command: command,
691-
Env: env,
692-
Image: devicePluginImage,
693-
ImagePullPolicy: ipp,
694-
Resources: resources,
695-
VolumeMounts: []v1.VolumeMount{dpVolMount},
677+
initContainer := &kmmv1beta1.DevicePluginContainerSpec{
678+
Args: args,
679+
Command: command,
680+
Env: env,
681+
Image: devicePluginImage,
682+
Resources: resources,
683+
VolumeMounts: []v1.VolumeMount{dpVolMount},
684+
}
685+
if !withInitContainer {
686+
initContainer = nil
687+
}
688+
689+
mod := kmmv1beta1.Module{
690+
TypeMeta: metav1.TypeMeta{
691+
APIVersion: kmmv1beta1.GroupVersion.String(),
692+
Kind: "Module",
693+
},
694+
ObjectMeta: metav1.ObjectMeta{
695+
Name: moduleName,
696+
Namespace: namespace,
697+
},
698+
Spec: kmmv1beta1.ModuleSpec{
699+
ModuleLoader: moduleLoader,
700+
DevicePlugin: &kmmv1beta1.DevicePluginSpec{
701+
InitContainer: initContainer,
702+
Container: kmmv1beta1.DevicePluginContainerSpec{
703+
Args: args,
704+
Command: command,
705+
Env: env,
706+
Image: devicePluginImage,
707+
ImagePullPolicy: ipp,
708+
Resources: resources,
709+
VolumeMounts: []v1.VolumeMount{dpVolMount},
710+
},
711+
ServiceAccountName: serviceAccountName,
712+
Volumes: []v1.Volume{dpVol},
696713
},
697-
ServiceAccountName: serviceAccountName,
698-
Volumes: []v1.Volume{dpVol},
714+
ImageRepoSecret: &repoSecret,
715+
Selector: map[string]string{"has-feature-x": "true"},
716+
Tolerations: []v1.Toleration{testToleration},
699717
},
700-
ImageRepoSecret: &repoSecret,
701-
Selector: map[string]string{"has-feature-x": "true"},
702-
Tolerations: []v1.Toleration{testToleration},
703-
},
704-
}
705-
ds := appsv1.DaemonSet{
706-
ObjectMeta: metav1.ObjectMeta{
707-
Name: dsName,
708-
Namespace: namespace,
709-
},
710-
}
711-
712-
err := dsc.setDevicePluginAsDesired(context.Background(), &ds, &mod)
713-
Expect(err).NotTo(HaveOccurred())
718+
}
719+
ds := appsv1.DaemonSet{
720+
ObjectMeta: metav1.ObjectMeta{
721+
Name: dsName,
722+
Namespace: namespace,
723+
},
724+
}
714725

715-
podLabels := map[string]string{constants.ModuleNameLabel: moduleName}
726+
err := dsc.setDevicePluginAsDesired(context.Background(), &ds, &mod)
727+
Expect(err).NotTo(HaveOccurred())
716728

717-
directory := v1.HostPathDirectory
718-
expected := appsv1.DaemonSet{
719-
ObjectMeta: metav1.ObjectMeta{
720-
Name: dsName,
721-
Namespace: namespace,
722-
Labels: podLabels,
723-
OwnerReferences: []metav1.OwnerReference{
724-
{
725-
APIVersion: mod.APIVersion,
726-
BlockOwnerDeletion: ptr.To(true),
727-
Controller: ptr.To(true),
728-
Kind: mod.Kind,
729-
Name: moduleName,
730-
UID: mod.UID,
729+
podLabels := map[string]string{constants.ModuleNameLabel: moduleName}
730+
731+
expectedInitContainer := []v1.Container{
732+
{
733+
Args: args,
734+
Command: command,
735+
Env: env,
736+
Image: devicePluginImage,
737+
Name: "device-plugin-init",
738+
Resources: resources,
739+
SecurityContext: &v1.SecurityContext{
740+
Privileged: ptr.To(true),
741+
},
742+
VolumeMounts: []v1.VolumeMount{
743+
dpVolMount,
731744
},
732745
},
733-
},
734-
Spec: appsv1.DaemonSetSpec{
735-
Selector: &metav1.LabelSelector{MatchLabels: podLabels},
736-
Template: v1.PodTemplateSpec{
737-
ObjectMeta: metav1.ObjectMeta{
738-
Labels: podLabels,
739-
Finalizers: []string{constants.NodeLabelerFinalizer},
746+
}
747+
748+
if !withInitContainer {
749+
expectedInitContainer = nil
750+
}
751+
directory := v1.HostPathDirectory
752+
expected := appsv1.DaemonSet{
753+
ObjectMeta: metav1.ObjectMeta{
754+
Name: dsName,
755+
Namespace: namespace,
756+
Labels: podLabels,
757+
OwnerReferences: []metav1.OwnerReference{
758+
{
759+
APIVersion: mod.APIVersion,
760+
BlockOwnerDeletion: ptr.To(true),
761+
Controller: ptr.To(true),
762+
Kind: mod.Kind,
763+
Name: moduleName,
764+
UID: mod.UID,
765+
},
740766
},
741-
Spec: v1.PodSpec{
742-
Containers: []v1.Container{
743-
{
744-
Args: args,
745-
Command: command,
746-
Env: env,
747-
Image: devicePluginImage,
748-
ImagePullPolicy: ipp,
749-
Name: "device-plugin",
750-
Resources: resources,
751-
SecurityContext: &v1.SecurityContext{
752-
Privileged: ptr.To(true),
753-
},
754-
VolumeMounts: []v1.VolumeMount{
755-
dpVolMount,
756-
{
757-
Name: "kubelet-device-plugins",
758-
MountPath: "/var/lib/kubelet/device-plugins",
767+
},
768+
Spec: appsv1.DaemonSetSpec{
769+
Selector: &metav1.LabelSelector{MatchLabels: podLabels},
770+
Template: v1.PodTemplateSpec{
771+
ObjectMeta: metav1.ObjectMeta{
772+
Labels: podLabels,
773+
Finalizers: []string{constants.NodeLabelerFinalizer},
774+
},
775+
Spec: v1.PodSpec{
776+
InitContainers: expectedInitContainer,
777+
Containers: []v1.Container{
778+
{
779+
Args: args,
780+
Command: command,
781+
Env: env,
782+
Image: devicePluginImage,
783+
ImagePullPolicy: ipp,
784+
Name: "device-plugin",
785+
Resources: resources,
786+
SecurityContext: &v1.SecurityContext{
787+
Privileged: ptr.To(true),
788+
},
789+
VolumeMounts: []v1.VolumeMount{
790+
dpVolMount,
791+
{
792+
Name: "kubelet-device-plugins",
793+
MountPath: "/var/lib/kubelet/device-plugins",
794+
},
759795
},
760796
},
761797
},
762-
},
763-
ImagePullSecrets: []v1.LocalObjectReference{repoSecret},
764-
NodeSelector: expectedNodeSelector,
765-
PriorityClassName: "system-node-critical",
766-
ServiceAccountName: serviceAccountName,
767-
Volumes: []v1.Volume{
768-
{
769-
Name: "kubelet-device-plugins",
770-
VolumeSource: v1.VolumeSource{
771-
HostPath: &v1.HostPathVolumeSource{
772-
Path: "/var/lib/kubelet/device-plugins",
773-
Type: &directory,
798+
ImagePullSecrets: []v1.LocalObjectReference{repoSecret},
799+
NodeSelector: expectedNodeSelector,
800+
PriorityClassName: "system-node-critical",
801+
ServiceAccountName: serviceAccountName,
802+
Volumes: []v1.Volume{
803+
{
804+
Name: "kubelet-device-plugins",
805+
VolumeSource: v1.VolumeSource{
806+
HostPath: &v1.HostPathVolumeSource{
807+
Path: "/var/lib/kubelet/device-plugins",
808+
Type: &directory,
809+
},
774810
},
775811
},
812+
dpVol,
776813
},
777-
dpVol,
814+
Tolerations: []v1.Toleration{testToleration},
778815
},
779-
Tolerations: []v1.Toleration{testToleration},
780816
},
781817
},
782-
},
783-
}
784-
Expect(
785-
cmp.Equal(expected, ds),
786-
).To(
787-
BeTrue(), cmp.Diff(expected, ds),
788-
)
789-
},
818+
}
819+
Expect(
820+
cmp.Equal(expected, ds),
821+
).To(
822+
BeTrue(), cmp.Diff(expected, ds),
823+
)
824+
},
790825
Entry("moduleLoader is nil",
791826
nil,
792827
map[string]string{"has-feature-x": "true"},
828+
false,
793829
),
794830
Entry("moduleLoader is defined",
795831
&kmmv1beta1.ModuleLoaderSpec{},
796832
map[string]string{utils.GetKernelModuleReadyNodeLabel(namespace, moduleName): ""},
833+
true,
797834
),
798835
)
799836
})

0 commit comments

Comments
 (0)