Skip to content

Commit 36ee6cf

Browse files
kobzonegakobzonega
authored
YDBOPS-7162 Immutable statefulSet selectorLabels with label "ydb.tech/statefulset-name" (#172)
1 parent 6c814de commit 36ee6cf

File tree

13 files changed

+194
-112
lines changed

13 files changed

+194
-112
lines changed

‎internal/controllers/database/sync.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -322,42 +322,32 @@ func (r *Reconciler) waitForStatefulSetToScale(
322322
r.Recorder.Event(
323323
database,
324324
corev1.EventTypeWarning,
325-
"ProvisioningFailed",
326-
fmt.Sprintf("Failed to get StatefulSets: %s", err),
325+
"ControllerError",
326+
fmt.Sprintf("Failed to get StatefulSet: %s", err),
327327
)
328328
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
329329
}
330330

331-
podLabels := labels.Common(database.Name, make(map[string]string))
332-
podLabels.Merge(map[string]string{
333-
labels.ComponentKey: labels.DynamicComponent,
334-
})
335-
336-
matchingLabels := client.MatchingLabels{}
337-
for k, v := range podLabels {
338-
matchingLabels[k] = v
339-
}
340-
341331
podList := &corev1.PodList{}
342332
opts := []client.ListOption{
343333
client.InNamespace(database.Namespace),
344-
matchingLabels,
334+
client.MatchingLabels{labels.StatefulsetComponent: database.Name},
345335
}
346336

347337
err = r.List(ctx, podList, opts...)
348338
if err != nil {
349339
r.Recorder.Event(
350340
database,
351341
corev1.EventTypeWarning,
352-
"ProvisioningFailed",
353-
fmt.Sprintf("Failed to list cluster pods: %s", err),
342+
"ControllerError",
343+
fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
354344
)
355345
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
356346
}
357347

358348
runningPods := 0
359349
for _, e := range podList.Items {
360-
if e.Status.Phase == "Running" {
350+
if resources.PodIsReady(e) {
361351
runningPods++
362352
}
363353
}
@@ -367,13 +357,13 @@ func (r *Reconciler) waitForStatefulSetToScale(
367357
database,
368358
corev1.EventTypeNormal,
369359
string(DatabaseProvisioning),
370-
fmt.Sprintf("Waiting for number of running dynamic pods to match expected: %d != %d", runningPods, database.Spec.Nodes),
360+
fmt.Sprintf("Waiting for number of running pods to match expected: %d != %d", runningPods, database.Spec.Nodes),
371361
)
372362
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
373363
Type: DatabaseProvisionedCondition,
374364
Status: metav1.ConditionFalse,
375365
Reason: ReasonInProgress,
376-
Message: fmt.Sprintf("Number of running dynamic pods does not match expected: %d != %d", runningPods, database.Spec.Nodes),
366+
Message: fmt.Sprintf("Number of running pods does not match expected: %d != %d", runningPods, database.Spec.Nodes),
377367
})
378368
return r.updateStatus(ctx, database, DefaultRequeueDelay)
379369
}
@@ -383,7 +373,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
383373
Type: DatabaseProvisionedCondition,
384374
Status: metav1.ConditionTrue,
385375
Reason: ReasonCompleted,
386-
Message: "Successfully scaled to desired number of nodes",
376+
Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", database.Spec.Nodes),
387377
})
388378
return r.updateStatus(ctx, database, StatusUpdateRequeueDelay)
389379
}

‎internal/controllers/databasenodeset/sync.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919

2020
"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
2121
. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
22+
"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
2223
"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
2324
)
2425

@@ -213,15 +214,15 @@ func (r *Reconciler) waitForStatefulSetToScale(
213214
r.Recorder.Event(
214215
databaseNodeSet,
215216
corev1.EventTypeWarning,
216-
"Syncing",
217-
fmt.Sprintf("Failed to found StatefulSet: %s", err),
217+
"ProvisioningFailed",
218+
fmt.Sprintf("StatefulSet with name %s was not found: %s", databaseNodeSet.Name, err),
218219
)
219220
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil
220221
}
221222
r.Recorder.Event(
222223
databaseNodeSet,
223224
corev1.EventTypeWarning,
224-
"Syncing",
225+
"ControllerError",
225226
fmt.Sprintf("Failed to get StatefulSets: %s", err),
226227
)
227228
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
@@ -235,21 +236,22 @@ func (r *Reconciler) waitForStatefulSetToScale(
235236
podList := &corev1.PodList{}
236237
opts := []client.ListOption{
237238
client.InNamespace(databaseNodeSet.Namespace),
238-
matchingLabels,
239+
client.MatchingLabels{labels.StatefulsetComponent: databaseNodeSet.Name},
239240
}
240-
if err = r.List(ctx, podList, opts...); err != nil {
241+
err = r.List(ctx, podList, opts...)
242+
if err != nil {
241243
r.Recorder.Event(
242244
databaseNodeSet,
243245
corev1.EventTypeWarning,
244-
"Syncing",
245-
fmt.Sprintf("Failed to list databaseNodeSet pods: %s", err),
246+
"ControllerError",
247+
fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
246248
)
247249
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
248250
}
249251

250252
runningPods := 0
251253
for _, e := range podList.Items {
252-
if e.Status.Phase == "Running" {
254+
if resources.PodIsReady(e) {
253255
runningPods++
254256
}
255257
}
@@ -259,7 +261,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
259261
databaseNodeSet,
260262
corev1.EventTypeNormal,
261263
string(DatabaseNodeSetProvisioning),
262-
fmt.Sprintf("Waiting for number of running databaseNodeSet pods to match expected: %d != %d", runningPods, databaseNodeSet.Spec.Nodes),
264+
fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, databaseNodeSet.Spec.Nodes),
263265
)
264266
meta.SetStatusCondition(&databaseNodeSet.Status.Conditions, metav1.Condition{
265267
Type: NodeSetProvisionedCondition,
@@ -275,7 +277,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
275277
Type: NodeSetProvisionedCondition,
276278
Status: metav1.ConditionTrue,
277279
Reason: ReasonCompleted,
278-
Message: fmt.Sprintf("Scaled DatabaseNodeSet to %d successfully", databaseNodeSet.Spec.Nodes),
280+
Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", databaseNodeSet.Spec.Nodes),
279281
})
280282
return r.updateStatus(ctx, databaseNodeSet, StatusUpdateRequeueDelay)
281283
}

‎internal/controllers/storage/controller_test.go‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"context"
55
"fmt"
66
"path/filepath"
7-
"strconv"
87
"strings"
98
"testing"
109

@@ -114,8 +113,7 @@ var _ = Describe("Storage controller medium tests", func() {
114113
}
115114
Expect(foundVolume).To(BeTrue())
116115

117-
By("Check that label and annotation propagated to pods...", func() {
118-
podLabels := storageSS.Spec.Template.Labels
116+
By("Check that configuration checksum annotation propagated to pods...", func() {
119117
podAnnotations := storageSS.Spec.Template.Annotations
120118

121119
foundStorage := v1alpha1.Storage{}
@@ -124,12 +122,6 @@ var _ = Describe("Storage controller medium tests", func() {
124122
Namespace: testobjects.YdbNamespace,
125123
}, &foundStorage)).Should(Succeed())
126124

127-
foundStorageGenerationLabel := false
128-
if podLabels[labels.StorageGeneration] == strconv.FormatInt(foundStorage.ObjectMeta.Generation, 10) {
129-
foundStorageGenerationLabel = true
130-
}
131-
Expect(foundStorageGenerationLabel).To(BeTrue())
132-
133125
foundConfigurationChecksumAnnotation := false
134126
if podAnnotations[annotations.ConfigurationChecksum] == resources.GetConfigurationChecksum(foundStorage.Spec.Configuration) {
135127
foundConfigurationChecksumAnnotation = true
@@ -150,5 +142,50 @@ var _ = Describe("Storage controller medium tests", func() {
150142
Expect(labelArgKey).Should(BeEquivalentTo(v1alpha1.LabelDeploymentKey))
151143
Expect(labelArgValue).Should(BeEquivalentTo(v1alpha1.LabelDeploymentValueKubernetes))
152144
})
145+
146+
By("Check that statefulset podTemplate labels remain immutable...", func() {
147+
testLabelKey := "ydb-label"
148+
testLabelValue := "test"
149+
By("set additional labels to Storage...")
150+
Eventually(func() error {
151+
foundStorage := v1alpha1.Storage{}
152+
Expect(k8sClient.Get(ctx, types.NamespacedName{
153+
Name: storageSample.Name,
154+
Namespace: testobjects.YdbNamespace,
155+
}, &foundStorage))
156+
additionalLabels := resources.CopyDict(foundStorage.Spec.AdditionalLabels)
157+
additionalLabels[testLabelKey] = testLabelValue
158+
foundStorage.Spec.AdditionalLabels = additionalLabels
159+
return k8sClient.Update(ctx, &foundStorage)
160+
}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())
161+
162+
By("check that additional labels was added...")
163+
foundStatefulSets := appsv1.StatefulSetList{}
164+
Eventually(func() error {
165+
err := k8sClient.List(ctx, &foundStatefulSets,
166+
client.InNamespace(testobjects.YdbNamespace),
167+
)
168+
if err != nil {
169+
return err
170+
}
171+
value, exist := foundStatefulSets.Items[0].Labels[testLabelKey]
172+
if !exist {
173+
return fmt.Errorf("label key `ydb-label` does not exist in StatefulSet. Current labels: %s", foundStatefulSets.Items[0].Labels)
174+
}
175+
if value != testLabelValue {
176+
return fmt.Errorf("label value `ydb-label` in StatefulSet does not equal `test`. Current labels: %s", foundStatefulSets.Items[0].Labels)
177+
}
178+
return nil
179+
}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())
180+
181+
By("check that StatefulSet selector was not updated...")
182+
Expect(*foundStatefulSets.Items[0].Spec.Selector).Should(BeEquivalentTo(
183+
metav1.LabelSelector{
184+
MatchLabels: map[string]string{
185+
labels.StatefulsetComponent: storageSample.Name,
186+
},
187+
},
188+
))
189+
})
153190
})
154191
})

‎internal/controllers/storage/sync.go‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -142,42 +142,32 @@ func (r *Reconciler) waitForStatefulSetToScale(
142142
r.Recorder.Event(
143143
storage,
144144
corev1.EventTypeWarning,
145-
"ProvisioningFailed",
146-
fmt.Sprintf("Failed to get StatefulSets: %s", err),
145+
"ControllerError",
146+
fmt.Sprintf("Failed to get StatefulSet: %s", err),
147147
)
148148
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
149149
}
150150

151-
podLabels := labels.Common(storage.Name, make(map[string]string))
152-
podLabels.Merge(map[string]string{
153-
labels.ComponentKey: labels.StorageComponent,
154-
})
155-
156-
matchingLabels := client.MatchingLabels{}
157-
for k, v := range podLabels {
158-
matchingLabels[k] = v
159-
}
160-
161151
podList := &corev1.PodList{}
162152
opts := []client.ListOption{
163153
client.InNamespace(storage.Namespace),
164-
matchingLabels,
154+
client.MatchingLabels{labels.StatefulsetComponent: storage.Name},
165155
}
166156

167157
err = r.List(ctx, podList, opts...)
168158
if err != nil {
169159
r.Recorder.Event(
170160
storage,
171161
corev1.EventTypeWarning,
172-
"ProvisioningFailed",
173-
fmt.Sprintf("Failed to list cluster pods: %s", err),
162+
"ControllerError",
163+
fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
174164
)
175165
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
176166
}
177167

178168
runningPods := 0
179169
for _, e := range podList.Items {
180-
if e.Status.Phase == "Running" {
170+
if resources.PodIsReady(e) {
181171
runningPods++
182172
}
183173
}
@@ -187,7 +177,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
187177
storage,
188178
corev1.EventTypeNormal,
189179
string(StorageProvisioning),
190-
fmt.Sprintf("Waiting for number of running storage pods to match expected: %d != %d", runningPods, storage.Spec.Nodes),
180+
fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, storage.Spec.Nodes),
191181
)
192182
meta.SetStatusCondition(&storage.Status.Conditions, metav1.Condition{
193183
Type: StorageProvisionedCondition,
@@ -203,7 +193,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
203193
Type: StorageProvisionedCondition,
204194
Status: metav1.ConditionTrue,
205195
Reason: ReasonCompleted,
206-
Message: "Successfully scaled to desired number of nodes",
196+
Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storage.Spec.Nodes),
207197
})
208198
return r.updateStatus(ctx, storage, StatusUpdateRequeueDelay)
209199
}

‎internal/controllers/storagenodeset/sync.go‎

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919

2020
"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
2121
. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants" //nolint:revive,stylecheck
22+
"github.com/ydb-platform/ydb-kubernetes-operator/internal/labels"
2223
"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
2324
)
2425

@@ -213,49 +214,40 @@ func (r *Reconciler) waitForStatefulSetToScale(
213214
r.Recorder.Event(
214215
storageNodeSet,
215216
corev1.EventTypeWarning,
216-
"Syncing",
217-
fmt.Sprintf("Failed to found StatefulSet: %s", err),
217+
"ProvisioningFailed",
218+
fmt.Sprintf("StatefulSet with name %s was not found: %s", storageNodeSet.Name, err),
218219
)
219220
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
220221
}
221222
r.Recorder.Event(
222223
storageNodeSet,
223224
corev1.EventTypeWarning,
224-
"Syncing",
225-
fmt.Sprintf("Failed to get StatefulSets: %s", err),
225+
"ControllerError",
226+
fmt.Sprintf("Failed to get StatefulSet: %s", err),
226227
)
227228
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
228229
}
229230

230-
matchingLabels := client.MatchingLabels{}
231-
for k, v := range storageNodeSet.Labels {
232-
matchingLabels[k] = v
233-
}
234-
235231
podList := &corev1.PodList{}
236232
opts := []client.ListOption{
237233
client.InNamespace(storageNodeSet.Namespace),
238-
matchingLabels,
234+
client.MatchingLabels{labels.StatefulsetComponent: storageNodeSet.Name},
239235
}
240-
if err = r.List(ctx, podList, opts...); err != nil {
236+
237+
err = r.List(ctx, podList, opts...)
238+
if err != nil {
241239
r.Recorder.Event(
242240
storageNodeSet,
243241
corev1.EventTypeWarning,
244-
"Syncing",
245-
fmt.Sprintf("Failed to list storageNodeSet pods: %s", err),
242+
"ControllerError",
243+
fmt.Sprintf("Failed to list StatefulSet pods: %s", err),
246244
)
247-
meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{
248-
Type: NodeSetProvisionedCondition,
249-
Status: metav1.ConditionFalse,
250-
Reason: ReasonInProgress,
251-
Message: "Failed to check Pods .status.phase",
252-
})
253245
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
254246
}
255247

256248
runningPods := 0
257249
for _, e := range podList.Items {
258-
if e.Status.Phase == "Running" {
250+
if resources.PodIsReady(e) {
259251
runningPods++
260252
}
261253
}
@@ -265,7 +257,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
265257
storageNodeSet,
266258
corev1.EventTypeNormal,
267259
string(StorageNodeSetProvisioning),
268-
fmt.Sprintf("Waiting for number of running storageNodeSet pods to match expected: %d != %d", runningPods, storageNodeSet.Spec.Nodes),
260+
fmt.Sprintf("Waiting for number of running nodes to match expected: %d != %d", runningPods, storageNodeSet.Spec.Nodes),
269261
)
270262
meta.SetStatusCondition(&storageNodeSet.Status.Conditions, metav1.Condition{
271263
Type: NodeSetProvisionedCondition,
@@ -281,7 +273,7 @@ func (r *Reconciler) waitForStatefulSetToScale(
281273
Type: NodeSetProvisionedCondition,
282274
Status: metav1.ConditionTrue,
283275
Reason: ReasonCompleted,
284-
Message: fmt.Sprintf("Scaled StorageNodeSet to %d successfully", storageNodeSet.Spec.Nodes),
276+
Message: fmt.Sprintf("Successfully scaled to desired number of nodes: %d", storageNodeSet.Spec.Nodes),
285277
})
286278
return r.updateStatus(ctx, storageNodeSet, StatusUpdateRequeueDelay)
287279
}

‎internal/labels/label.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ const (
1919

2020
// ServiceComponent The specialization of a Service resource
2121
ServiceComponent = "ydb.tech/service-for"
22+
// StatefulsetComponent The specialization of a Statefulset resource
23+
StatefulsetComponent = "ydb.tech/statefulset-name"
2224
// StorageNodeSetComponent The specialization of a StorageNodeSet resource
2325
StorageNodeSetComponent = "ydb.tech/storage-nodeset"
2426
// DatabaseNodeSetComponent The specialization of a DatabaseNodeSet resource

0 commit comments

Comments
 (0)