Skip to content

Commit 9343f17

Browse files
yevgeny-shnaidmank8s-ci-robot
yevgeny-shnaidman
authored and
k8s-ci-robot
committed
Adding "version.ready" label to the ordered upgrade flow
Currently we have only one "ready" label to signify that the kernel module is loaded successfuly. In order upgrade scenario, the user needs to know that the loaded module is actually the one with the targeted module version, in order to coordinated further actions. This PR introduces a new label: kmm.node.kubernetes.io/%s.%s.version.ready, whose value will signify the version of the currently loaded kernel module. The changes in the code are: 1. Adding Version field both to status and spec of NMC (will be used to construct the new label) 2. Polulating the Version field of the NCM's spec during NMC update/create 3. Adding the "version" annotation to the worker pod, so that the version may be extracted for NMC's status 4. Changing the Node package API to receive maps instead of slice for labels to be added/remove. This is done since now we must also update the value of the label, and not just the key 5. Rewrite the UpdateNodeLabel function in the NMC controller to support new flow and new Node API 6. In case "ready" label is removed, we also automaticaly remove "version.ready" label. If it does not exists - nothing happens 7. In case "ready" label needs to be added, we check if the version field exists in the NMC status, and if it does, then we add the "version.ready" label with appropriate value
1 parent 02563b9 commit 9343f17

File tree

13 files changed

+226
-120
lines changed

13 files changed

+226
-120
lines changed

‎api/v1beta1/nodemodulesconfig_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ type ModuleItem struct {
4444
//+optional
4545
// tolerations define which tolerations should be added for every load/unload pod running on the node
4646
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
47+
//+optional
48+
// Version is the version of the kernel module that should be loaded
49+
Version string `json:"version,omitempty"`
4750
}
4851

4952
type NodeModuleSpec struct {

‎config/crd-hub/bases/kmm.sigs.x-k8s.io_nodemodulesconfigs.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,10 @@ spec:
222222
type: string
223223
type: object
224224
type: array
225+
version:
226+
description: Version is the version of the kernel module that
227+
should be loaded
228+
type: string
225229
required:
226230
- config
227231
- name
@@ -415,6 +419,10 @@ spec:
415419
type: string
416420
type: object
417421
type: array
422+
version:
423+
description: Version is the version of the kernel module that
424+
should be loaded
425+
type: string
418426
required:
419427
- name
420428
- namespace

‎config/crd/bases/kmm.sigs.x-k8s.io_nodemodulesconfigs.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,10 @@ spec:
222222
type: string
223223
type: object
224224
type: array
225+
version:
226+
description: Version is the version of the kernel module that
227+
should be loaded
228+
type: string
225229
required:
226230
- config
227231
- name
@@ -415,6 +419,10 @@ spec:
415419
type: string
416420
type: object
417421
type: array
422+
version:
423+
description: Version is the version of the kernel module that
424+
should be loaded
425+
type: string
418426
required:
419427
- name
420428
- namespace

‎internal/controllers/mock_nmc_reconciler.go‎

Lines changed: 16 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/controllers/nmc_reconciler.go‎

Lines changed: 88 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -102,15 +102,15 @@ func (r *NMCReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
102102
}
103103

104104
errs := make([]error, 0, len(nmcObj.Spec.Modules)+len(nmcObj.Status.Modules))
105-
var readyLabelsToRemove []string
105+
readyLabelsToRemove := make(map[string]string)
106106
for _, mod := range nmcObj.Spec.Modules {
107107
moduleNameKey := mod.Namespace + "/" + mod.Name
108108

109109
logger := logger.WithValues("module", moduleNameKey)
110110

111111
// skipping handling NMC spec module until node is ready
112112
if !r.nodeAPI.IsNodeSchedulable(&node, mod.Tolerations) {
113-
readyLabelsToRemove = append(readyLabelsToRemove, utils.GetKernelModuleReadyNodeLabel(mod.Namespace, mod.Name))
113+
readyLabelsToRemove[utils.GetKernelModuleReadyNodeLabel(mod.Namespace, mod.Name)] = ""
114114
delete(statusMap, moduleNameKey)
115115
continue
116116
}
@@ -141,7 +141,7 @@ func (r *NMCReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r
141141
}
142142

143143
// removing label of loaded kmods
144-
if readyLabelsToRemove != nil {
144+
if len(readyLabelsToRemove) != 0 {
145145
if err := r.nodeAPI.UpdateLabels(ctx, &node, nil, readyLabelsToRemove); err != nil {
146146
return ctrl.Result{}, fmt.Errorf("could remove node %s labels: %v", node.Name, err)
147147
}
@@ -574,6 +574,8 @@ func (h *nmcReconcilerHelperImpl) SyncStatus(ctx context.Context, nmcObj *kmmv1b
574574

575575
status.BootId = node.Status.NodeInfo.BootID
576576

577+
status.Version = h.podManager.GetModuleVersionAnnotation(&p)
578+
577579
nmc.SetModuleStatus(&nmcObj.Status.Modules, *status)
578580

579581
podsToDelete = append(podsToDelete, p)
@@ -599,63 +601,6 @@ func (h *nmcReconcilerHelperImpl) SyncStatus(ctx context.Context, nmcObj *kmmv1b
599601
return errors.Join(errs...)
600602
}
601603

602-
type labelPreparationHelper interface {
603-
getDeprecatedKernelModuleReadyLabels(node v1.Node) sets.Set[string]
604-
getNodeKernelModuleReadyLabels(node v1.Node) sets.Set[types.NamespacedName]
605-
getSpecLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig
606-
getStatusLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig
607-
addEqualLabels(nodeModuleReadyLabels sets.Set[types.NamespacedName],
608-
specLabels, statusLabels map[types.NamespacedName]kmmv1beta1.ModuleConfig) []types.NamespacedName
609-
removeOrphanedLabels(nodeModuleReadyLabels sets.Set[types.NamespacedName],
610-
specLabels, statusLabels map[types.NamespacedName]kmmv1beta1.ModuleConfig) []types.NamespacedName
611-
}
612-
type labelPreparationHelperImpl struct{}
613-
614-
func newLabelPreparationHelper() labelPreparationHelper {
615-
return &labelPreparationHelperImpl{}
616-
}
617-
618-
func (lph *labelPreparationHelperImpl) getNodeKernelModuleReadyLabels(node v1.Node) sets.Set[types.NamespacedName] {
619-
nodeModuleReadyLabels := sets.New[types.NamespacedName]()
620-
621-
for label := range node.GetLabels() {
622-
if ok, namespace, name := utils.IsKernelModuleReadyNodeLabel(label); ok {
623-
nodeModuleReadyLabels.Insert(types.NamespacedName{Namespace: namespace, Name: name})
624-
}
625-
}
626-
return nodeModuleReadyLabels
627-
}
628-
629-
func (lph *labelPreparationHelperImpl) getDeprecatedKernelModuleReadyLabels(node v1.Node) sets.Set[string] {
630-
deprecatedNodeModuleReadyLabels := sets.New[string]()
631-
632-
for label := range node.GetLabels() {
633-
if utils.IsDeprecatedKernelModuleReadyNodeLabel(label) {
634-
deprecatedNodeModuleReadyLabels.Insert(label)
635-
}
636-
}
637-
return deprecatedNodeModuleReadyLabels
638-
}
639-
640-
func (lph *labelPreparationHelperImpl) getSpecLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig {
641-
specLabels := make(map[types.NamespacedName]kmmv1beta1.ModuleConfig)
642-
643-
for _, module := range nmc.Spec.Modules {
644-
specLabels[types.NamespacedName{Namespace: module.Namespace, Name: module.Name}] = module.Config
645-
}
646-
return specLabels
647-
}
648-
649-
func (lph *labelPreparationHelperImpl) getStatusLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig {
650-
statusLabels := make(map[types.NamespacedName]kmmv1beta1.ModuleConfig)
651-
652-
for _, module := range nmc.Status.Modules {
653-
label := types.NamespacedName{Namespace: module.Namespace, Name: module.Name}
654-
statusLabels[label] = module.Config
655-
}
656-
return statusLabels
657-
}
658-
659604
func (h *nmcReconcilerHelperImpl) UpdateNodeLabels(ctx context.Context, nmc *kmmv1beta1.NodeModulesConfig, node *v1.Node) ([]types.NamespacedName, []types.NamespacedName, error) {
660605

661606
// get all the kernel module ready labels of the node
@@ -668,20 +613,30 @@ func (h *nmcReconcilerHelperImpl) UpdateNodeLabels(ctx context.Context, nmc *kmm
668613
// get status labels and their config
669614
statusLabels := h.lph.getStatusLabelsAndTheirConfigs(nmc)
670615

616+
// get the versions per the name/namespace of the module
617+
statusVersions := h.lph.getStatusVersions(nmc)
618+
671619
// label in node but not in spec or status - should be removed
672620
nsnLabelsToBeRemoved := h.lph.removeOrphanedLabels(nodeModuleReadyLabels, specLabels, statusLabels)
673621

674622
// label in spec and status and config equal - should be added
675623
nsnLabelsToBeLoaded := h.lph.addEqualLabels(nodeModuleReadyLabels, specLabels, statusLabels)
676624

677-
var loadedLabels []string
678-
unloadedLabels := deprecatedNodeModuleReadyLabels.UnsortedList()
625+
loadedLabels := make(map[string]string)
626+
unloadedLabels := deprecatedNodeModuleReadyLabels
679627

680628
for _, label := range nsnLabelsToBeRemoved {
681-
unloadedLabels = append(unloadedLabels, utils.GetKernelModuleReadyNodeLabel(label.Namespace, label.Name))
629+
unloadedLabels[utils.GetKernelModuleReadyNodeLabel(label.Namespace, label.Name)] = ""
630+
// unload the kernel ready version label also. if it does not exists, that's ok, the code won't fail.It also means
631+
// that the version label will not be in labelsToAdd, since status and spec are missing
632+
unloadedLabels[utils.GetKernelModuleVersionReadyNodeLabel(label.Namespace, label.Name)] = ""
682633
}
634+
683635
for _, label := range nsnLabelsToBeLoaded {
684-
loadedLabels = append(loadedLabels, utils.GetKernelModuleReadyNodeLabel(label.Namespace, label.Name))
636+
loadedLabels[utils.GetKernelModuleReadyNodeLabel(label.Namespace, label.Name)] = ""
637+
if version, ok := statusVersions[label]; ok {
638+
loadedLabels[utils.GetKernelModuleVersionReadyNodeLabel(label.Namespace, label.Name)] = version
639+
}
685640
}
686641

687642
if err := h.nodeAPI.UpdateLabels(ctx, node, loadedLabels, unloadedLabels); err != nil {
@@ -714,6 +669,75 @@ func (h *nmcReconcilerHelperImpl) RecordEvents(node *v1.Node, loadedModules, unl
714669
}
715670
}
716671

672+
type labelPreparationHelper interface {
673+
getDeprecatedKernelModuleReadyLabels(node v1.Node) map[string]string
674+
getNodeKernelModuleReadyLabels(node v1.Node) sets.Set[types.NamespacedName]
675+
getSpecLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig
676+
getStatusLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig
677+
getStatusVersions(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]string
678+
addEqualLabels(nodeModuleReadyLabels sets.Set[types.NamespacedName],
679+
specLabels, statusLabels map[types.NamespacedName]kmmv1beta1.ModuleConfig) []types.NamespacedName
680+
removeOrphanedLabels(nodeModuleReadyLabels sets.Set[types.NamespacedName],
681+
specLabels, statusLabels map[types.NamespacedName]kmmv1beta1.ModuleConfig) []types.NamespacedName
682+
}
683+
type labelPreparationHelperImpl struct{}
684+
685+
func newLabelPreparationHelper() labelPreparationHelper {
686+
return &labelPreparationHelperImpl{}
687+
}
688+
689+
func (lph *labelPreparationHelperImpl) getNodeKernelModuleReadyLabels(node v1.Node) sets.Set[types.NamespacedName] {
690+
nodeModuleReadyLabels := sets.New[types.NamespacedName]()
691+
692+
for label := range node.GetLabels() {
693+
if ok, namespace, name := utils.IsKernelModuleReadyNodeLabel(label); ok {
694+
nodeModuleReadyLabels.Insert(types.NamespacedName{Namespace: namespace, Name: name})
695+
}
696+
}
697+
return nodeModuleReadyLabels
698+
}
699+
700+
func (lph *labelPreparationHelperImpl) getDeprecatedKernelModuleReadyLabels(node v1.Node) map[string]string {
701+
deprecatedLabels := make(map[string]string)
702+
703+
for key, val := range node.GetLabels() {
704+
if utils.IsDeprecatedKernelModuleReadyNodeLabel(key) {
705+
deprecatedLabels[key] = val
706+
}
707+
}
708+
return deprecatedLabels
709+
}
710+
711+
func (lph *labelPreparationHelperImpl) getSpecLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig {
712+
specLabels := make(map[types.NamespacedName]kmmv1beta1.ModuleConfig)
713+
714+
for _, module := range nmc.Spec.Modules {
715+
specLabels[types.NamespacedName{Namespace: module.Namespace, Name: module.Name}] = module.Config
716+
}
717+
return specLabels
718+
}
719+
720+
func (lph *labelPreparationHelperImpl) getStatusLabelsAndTheirConfigs(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]kmmv1beta1.ModuleConfig {
721+
statusLabels := make(map[types.NamespacedName]kmmv1beta1.ModuleConfig)
722+
723+
for _, module := range nmc.Status.Modules {
724+
label := types.NamespacedName{Namespace: module.Namespace, Name: module.Name}
725+
statusLabels[label] = module.Config
726+
}
727+
return statusLabels
728+
}
729+
730+
func (lph *labelPreparationHelperImpl) getStatusVersions(nmc *kmmv1beta1.NodeModulesConfig) map[types.NamespacedName]string {
731+
versions := make(map[types.NamespacedName]string)
732+
733+
for _, module := range nmc.Status.Modules {
734+
if module.Version != "" {
735+
versions[types.NamespacedName{Namespace: module.Namespace, Name: module.Name}] = module.Version
736+
}
737+
}
738+
return versions
739+
}
740+
717741
func (lph *labelPreparationHelperImpl) removeOrphanedLabels(nodeModuleReadyLabels sets.Set[types.NamespacedName],
718742
specLabels, statusLabels map[types.NamespacedName]kmmv1beta1.ModuleConfig) []types.NamespacedName {
719743

0 commit comments

Comments
 (0)