Skip to content

Commit 29b6433

Browse files
tsprasannaak8s-ci-robot
tsprasannaa
authored and
k8s-ci-robot
committed
Add support for user defined tolerances to modules
There is a requirement to run selected pods on a cordoned node. One such example is device driver upgrade. The procedure involves the node to be cordoned. At the same time, there should be a way to run house keeping pods that carry out the driver upgrades. These pods should be schedulable. This PR adds support to carry the tolerances for the pods that carry out the house keeping operations. These user defined tolerances will match the taint that gets added to the nodes. ModuleSpec will be used to carry the tolerance to the pods which will be used during pod creation
1 parent 4ed5021 commit 29b6433

21 files changed

+303
-57
lines changed

‎api/v1beta1/module_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,10 @@ type ModuleSpec struct {
328328

329329
// Selector describes on which nodes the Module should be loaded and optionally built.
330330
Selector map[string]string `json:"selector"`
331+
332+
// If specified, the pod's tolerations.
333+
// +optional
334+
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
331335
}
332336

333337
// DaemonSetStatus contains the status for a daemonset deployed during

‎api/v1beta1/nodemodulesconfig_types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ type ModuleConfig struct {
3333
//+optional
3434
InTreeModuleToRemove string `json:"inTreeModuleToRemove,omitempty"`
3535
Modprobe ModprobeSpec `json:"modprobe"`
36+
//+optional
37+
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
3638
}
3739

3840
type ModuleItem struct {

‎api/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd-hub/bases/hub.kmm.sigs.x-k8s.io_managedclustermodules.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2666,6 +2666,45 @@ spec:
26662666
description: Selector describes on which nodes the Module should
26672667
be loaded and optionally built.
26682668
type: object
2669+
tolerations:
2670+
description: If specified, the pod's tolerations.
2671+
items:
2672+
description: |-
2673+
The pod this Toleration is attached to tolerates any taint that matches
2674+
the triple <key,value,effect> using the matching operator <operator>.
2675+
properties:
2676+
effect:
2677+
description: |-
2678+
Effect indicates the taint effect to match. Empty means match all taint effects.
2679+
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
2680+
type: string
2681+
key:
2682+
description: |-
2683+
Key is the taint key that the toleration applies to. Empty means match all taint keys.
2684+
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
2685+
type: string
2686+
operator:
2687+
description: |-
2688+
Operator represents a key's relationship to the value.
2689+
Valid operators are Exists and Equal. Defaults to Equal.
2690+
Exists is equivalent to wildcard for value, so that a pod can
2691+
tolerate all taints of a particular category.
2692+
type: string
2693+
tolerationSeconds:
2694+
description: |-
2695+
TolerationSeconds represents the period of time the toleration (which must be
2696+
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
2697+
it is not set, which means tolerate the taint forever (do not evict). Zero and
2698+
negative values will be treated as 0 (evict immediately) by the system.
2699+
format: int64
2700+
type: integer
2701+
value:
2702+
description: |-
2703+
Value is the taint value the toleration matches to.
2704+
If the operator is Exists, the value should be empty, otherwise just a regular string.
2705+
type: string
2706+
type: object
2707+
type: array
26692708
required:
26702709
- moduleLoader
26712710
- selector

‎config/crd/bases/kmm.sigs.x-k8s.io_modules.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2642,6 +2642,45 @@ spec:
26422642
description: Selector describes on which nodes the Module should be
26432643
loaded and optionally built.
26442644
type: object
2645+
tolerations:
2646+
description: If specified, the pod's tolerations.
2647+
items:
2648+
description: |-
2649+
The pod this Toleration is attached to tolerates any taint that matches
2650+
the triple <key,value,effect> using the matching operator <operator>.
2651+
properties:
2652+
effect:
2653+
description: |-
2654+
Effect indicates the taint effect to match. Empty means match all taint effects.
2655+
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
2656+
type: string
2657+
key:
2658+
description: |-
2659+
Key is the taint key that the toleration applies to. Empty means match all taint keys.
2660+
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
2661+
type: string
2662+
operator:
2663+
description: |-
2664+
Operator represents a key's relationship to the value.
2665+
Valid operators are Exists and Equal. Defaults to Equal.
2666+
Exists is equivalent to wildcard for value, so that a pod can
2667+
tolerate all taints of a particular category.
2668+
type: string
2669+
tolerationSeconds:
2670+
description: |-
2671+
TolerationSeconds represents the period of time the toleration (which must be
2672+
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
2673+
it is not set, which means tolerate the taint forever (do not evict). Zero and
2674+
negative values will be treated as 0 (evict immediately) by the system.
2675+
format: int64
2676+
type: integer
2677+
value:
2678+
description: |-
2679+
Value is the taint value the toleration matches to.
2680+
If the operator is Exists, the value should be empty, otherwise just a regular string.
2681+
type: string
2682+
type: object
2683+
type: array
26452684
required:
26462685
- moduleLoader
26472686
- selector

‎config/crd/bases/kmm.sigs.x-k8s.io_nodemodulesconfigs.yaml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,44 @@ spec:
154154
type: array
155155
type: object
156156
type: object
157+
tolerations:
158+
items:
159+
description: |-
160+
The pod this Toleration is attached to tolerates any taint that matches
161+
the triple <key,value,effect> using the matching operator <operator>.
162+
properties:
163+
effect:
164+
description: |-
165+
Effect indicates the taint effect to match. Empty means match all taint effects.
166+
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
167+
type: string
168+
key:
169+
description: |-
170+
Key is the taint key that the toleration applies to. Empty means match all taint keys.
171+
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
172+
type: string
173+
operator:
174+
description: |-
175+
Operator represents a key's relationship to the value.
176+
Valid operators are Exists and Equal. Defaults to Equal.
177+
Exists is equivalent to wildcard for value, so that a pod can
178+
tolerate all taints of a particular category.
179+
type: string
180+
tolerationSeconds:
181+
description: |-
182+
TolerationSeconds represents the period of time the toleration (which must be
183+
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
184+
it is not set, which means tolerate the taint forever (do not evict). Zero and
185+
negative values will be treated as 0 (evict immediately) by the system.
186+
format: int64
187+
type: integer
188+
value:
189+
description: |-
190+
Value is the taint value the toleration matches to.
191+
If the operator is Exists, the value should be empty, otherwise just a regular string.
192+
type: string
193+
type: object
194+
type: array
157195
required:
158196
- containerImage
159197
- imagePullPolicy
@@ -306,6 +344,44 @@ spec:
306344
type: array
307345
type: object
308346
type: object
347+
tolerations:
348+
items:
349+
description: |-
350+
The pod this Toleration is attached to tolerates any taint that matches
351+
the triple <key,value,effect> using the matching operator <operator>.
352+
properties:
353+
effect:
354+
description: |-
355+
Effect indicates the taint effect to match. Empty means match all taint effects.
356+
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
357+
type: string
358+
key:
359+
description: |-
360+
Key is the taint key that the toleration applies to. Empty means match all taint keys.
361+
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
362+
type: string
363+
operator:
364+
description: |-
365+
Operator represents a key's relationship to the value.
366+
Valid operators are Exists and Equal. Defaults to Equal.
367+
Exists is equivalent to wildcard for value, so that a pod can
368+
tolerate all taints of a particular category.
369+
type: string
370+
tolerationSeconds:
371+
description: |-
372+
TolerationSeconds represents the period of time the toleration (which must be
373+
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
374+
it is not set, which means tolerate the taint forever (do not evict). Zero and
375+
negative values will be treated as 0 (evict immediately) by the system.
376+
format: int64
377+
type: integer
378+
value:
379+
description: |-
380+
Value is the taint value the toleration matches to.
381+
If the operator is Exists, the value should be empty, otherwise just a regular string.
382+
type: string
383+
type: object
384+
type: array
309385
required:
310386
- containerImage
311387
- imagePullPolicy

‎internal/api/moduleloaderdata.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ type ModuleLoaderData struct {
6262

6363
// used for setting the owner field of pods/buildconfigs
6464
Owner metav1.Object
65+
66+
// If specified, the pod's tolerations.
67+
// +optional
68+
Tolerations []v1.Toleration `json:"tolerations,omitempty"`
6569
}
6670

6771
func (mld *ModuleLoaderData) NamespacedName() types.NamespacedName {

‎internal/build/pod/maker.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ func (m *maker) podSpec(mld *api.ModuleLoaderData, containerImage string, pushIm
134134
RestartPolicy: v1.RestartPolicyNever,
135135
Volumes: volumes(mld.ImageRepoSecret, buildConfig),
136136
NodeSelector: selector,
137+
Tolerations: mld.Tolerations,
137138
}
138139
}
139140

‎internal/controllers/build_sign_reconciler.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func (r *BuildSignReconciler) Reconcile(ctx context.Context, mod *kmmv1beta1.Mod
7676
res := ctrl.Result{}
7777

7878
logger := log.FromContext(ctx)
79-
targetedNodes, err := r.nodeAPI.GetNodesListBySelector(ctx, mod.Spec.Selector)
79+
targetedNodes, err := r.nodeAPI.GetNodesListBySelector(ctx, mod.Spec.Selector, mod.Spec.Tolerations)
8080
if err != nil {
8181
return res, fmt.Errorf("could get targeted nodes for module %s: %w", mod.Name, err)
8282
}

‎internal/controllers/build_sign_reconciler_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ var _ = Describe("BuildSignReconciler_Reconcile", func() {
5555
mappings := map[string]*api.ModuleLoaderData{"kernelVersion": &api.ModuleLoaderData{}}
5656
returnedError := fmt.Errorf("some error")
5757
if getNodesError {
58-
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector).Return(nil, returnedError)
58+
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector, nil).Return(nil, returnedError)
5959
goto executeTestFunction
6060
}
61-
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector).Return(selectNodesList, nil)
61+
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector, nil).Return(selectNodesList, nil)
6262
if getMappingsError {
6363
mockReconHelper.EXPECT().getRelevantKernelMappings(ctx, &mod, selectNodesList).Return(nil, returnedError)
6464
goto executeTestFunction
@@ -93,7 +93,7 @@ var _ = Describe("BuildSignReconciler_Reconcile", func() {
9393
selectNodesList := []v1.Node{v1.Node{}}
9494
mappings := map[string]*api.ModuleLoaderData{"kernelVersion": &api.ModuleLoaderData{}}
9595
gomock.InOrder(
96-
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector).Return(selectNodesList, nil),
96+
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector, nil).Return(selectNodesList, nil),
9797
mockReconHelper.EXPECT().getRelevantKernelMappings(ctx, mod, selectNodesList).Return(mappings, nil),
9898
mockReconHelper.EXPECT().handleBuild(ctx, mappings["kernelVersion"]).Return(false, nil),
9999
mockReconHelper.EXPECT().garbageCollect(ctx, mod, mappings).Return(nil),
@@ -109,7 +109,7 @@ var _ = Describe("BuildSignReconciler_Reconcile", func() {
109109
selectNodesList := []v1.Node{v1.Node{}}
110110
mappings := map[string]*api.ModuleLoaderData{"kernelVersion": &api.ModuleLoaderData{}}
111111
gomock.InOrder(
112-
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector).Return(selectNodesList, nil),
112+
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector, nil).Return(selectNodesList, nil),
113113
mockReconHelper.EXPECT().getRelevantKernelMappings(ctx, mod, selectNodesList).Return(mappings, nil),
114114
mockReconHelper.EXPECT().handleBuild(ctx, mappings["kernelVersion"]).Return(true, nil),
115115
mockReconHelper.EXPECT().handleSigning(ctx, mappings["kernelVersion"]).Return(false, nil),
@@ -127,7 +127,7 @@ var _ = Describe("BuildSignReconciler_Reconcile", func() {
127127
selectNodesList := []v1.Node{v1.Node{}}
128128
mappings := map[string]*api.ModuleLoaderData{"kernelVersion": &api.ModuleLoaderData{}}
129129
gomock.InOrder(
130-
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector).Return(selectNodesList, nil),
130+
mn.EXPECT().GetNodesListBySelector(ctx, mod.Spec.Selector, nil).Return(selectNodesList, nil),
131131
mockReconHelper.EXPECT().getRelevantKernelMappings(ctx, mod, selectNodesList).Return(mappings, nil),
132132
mockReconHelper.EXPECT().handleBuild(ctx, mappings["kernelVersion"]).Return(true, nil),
133133
mockReconHelper.EXPECT().handleSigning(ctx, mappings["kernelVersion"]).Return(true, nil),

0 commit comments

Comments
 (0)