Skip to content

Commit 4b26289

Browse files
jottofarjottofar
committed
Bug 1884334: UpdateError: add and use Progressing field
1 parent 84b3884 commit 4b26289

File tree

5 files changed

+141
-93
lines changed

5 files changed

+141
-93
lines changed

‎pkg/cvo/internal/operatorstatus.go

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,11 @@ func waitForOperatorStatusToBeDone(ctx context.Context, interval time.Duration,
128128
actual, err := client.Get(ctx, expected.Name)
129129
if err != nil {
130130
lastErr = &payload.UpdateError{
131-
Nested: err,
132-
Reason: "ClusterOperatorNotAvailable",
133-
Message: fmt.Sprintf("Cluster operator %s has not yet reported success", expected.Name),
134-
Name: expected.Name,
131+
Nested: err,
132+
Progressing: true,
133+
Reason: "ClusterOperatorNotAvailable",
134+
Message: fmt.Sprintf("Cluster operator %s has not yet reported success", expected.Name),
135+
Name: expected.Name,
135136
}
136137
return false, nil
137138
}
@@ -160,10 +161,11 @@ func waitForOperatorStatusToBeDone(ctx context.Context, interval time.Duration,
160161

161162
message := fmt.Sprintf("Cluster operator %s is still updating", actual.Name)
162163
lastErr = &payload.UpdateError{
163-
Nested: errors.New(lowerFirst(message)),
164-
Reason: "ClusterOperatorNotAvailable",
165-
Message: message,
166-
Name: actual.Name,
164+
Nested: errors.New(lowerFirst(message)),
165+
Progressing: true,
166+
Reason: "ClusterOperatorNotAvailable",
167+
Message: message,
168+
Name: actual.Name,
167169
}
168170
return false, nil
169171
}
@@ -220,10 +222,11 @@ func waitForOperatorStatusToBeDone(ctx context.Context, interval time.Duration,
220222
message = fmt.Sprintf("Cluster operator %s is reporting a failure: %s", actual.Name, condition.Message)
221223
}
222224
lastErr = &payload.UpdateError{
223-
Nested: errors.New(lowerFirst(message)),
224-
Reason: "ClusterOperatorDegraded",
225-
Message: message,
226-
Name: actual.Name,
225+
Nested: errors.New(lowerFirst(message)),
226+
Progressing: false,
227+
Reason: "ClusterOperatorDegraded",
228+
Message: message,
229+
Name: actual.Name,
227230
}
228231
return false, nil
229232
}
@@ -232,9 +235,10 @@ func waitForOperatorStatusToBeDone(ctx context.Context, interval time.Duration,
232235
Nested: fmt.Errorf("cluster operator %s is not done; it is available=%v, progressing=%v, degraded=%v",
233236
actual.Name, available, progressing, degraded,
234237
),
235-
Reason: "ClusterOperatorNotAvailable",
236-
Message: fmt.Sprintf("Cluster operator %s has not yet reported success", actual.Name),
237-
Name: actual.Name,
238+
Progressing: progressing,
239+
Reason: "ClusterOperatorNotAvailable",
240+
Message: fmt.Sprintf("Cluster operator %s has not yet reported success", actual.Name),
241+
Name: actual.Name,
238242
}
239243
return false, nil
240244
}, ctx.Done())

‎pkg/cvo/internal/operatorstatus_test.go

Lines changed: 75 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
4343
},
4444
},
4545
expErr: &payload.UpdateError{
46-
Nested: apierrors.NewNotFound(schema.GroupResource{"", "clusteroperator"}, "test-co"),
47-
Reason: "ClusterOperatorNotAvailable",
48-
Message: "Cluster operator test-co has not yet reported success",
49-
Name: "test-co",
46+
Nested: apierrors.NewNotFound(schema.GroupResource{"", "clusteroperator"}, "test-co"),
47+
Progressing: true,
48+
Reason: "ClusterOperatorNotAvailable",
49+
Message: "Cluster operator test-co has not yet reported success",
50+
Name: "test-co",
5051
},
5152
}, {
5253
name: "cluster operator reporting no versions with no operands",
@@ -62,10 +63,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
6263
},
6364
},
6465
expErr: &payload.UpdateError{
65-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
66-
Reason: "ClusterOperatorNotAvailable",
67-
Message: "Cluster operator test-co is still updating",
68-
Name: "test-co",
66+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
67+
Progressing: true,
68+
Reason: "ClusterOperatorNotAvailable",
69+
Message: "Cluster operator test-co is still updating",
70+
Name: "test-co",
6971
},
7072
}, {
7173
name: "cluster operator reporting no versions",
@@ -83,10 +85,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
8385
},
8486
},
8587
expErr: &payload.UpdateError{
86-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
87-
Reason: "ClusterOperatorNotAvailable",
88-
Message: "Cluster operator test-co is still updating",
89-
Name: "test-co",
88+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
89+
Progressing: true,
90+
Reason: "ClusterOperatorNotAvailable",
91+
Message: "Cluster operator test-co is still updating",
92+
Name: "test-co",
9093
},
9194
}, {
9295
name: "cluster operator reporting no versions for operand",
@@ -109,10 +112,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
109112
},
110113
},
111114
expErr: &payload.UpdateError{
112-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
113-
Reason: "ClusterOperatorNotAvailable",
114-
Message: "Cluster operator test-co is still updating",
115-
Name: "test-co",
115+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
116+
Progressing: true,
117+
Reason: "ClusterOperatorNotAvailable",
118+
Message: "Cluster operator test-co is still updating",
119+
Name: "test-co",
116120
},
117121
}, {
118122
name: "cluster operator reporting old versions",
@@ -137,10 +141,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
137141
},
138142
},
139143
expErr: &payload.UpdateError{
140-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
141-
Reason: "ClusterOperatorNotAvailable",
142-
Message: "Cluster operator test-co is still updating",
143-
Name: "test-co",
144+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
145+
Progressing: true,
146+
Reason: "ClusterOperatorNotAvailable",
147+
Message: "Cluster operator test-co is still updating",
148+
Name: "test-co",
144149
},
145150
}, {
146151
name: "cluster operator reporting mix of desired and old versions",
@@ -165,10 +170,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
165170
},
166171
},
167172
expErr: &payload.UpdateError{
168-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
169-
Reason: "ClusterOperatorNotAvailable",
170-
Message: "Cluster operator test-co is still updating",
171-
Name: "test-co",
173+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
174+
Progressing: true,
175+
Reason: "ClusterOperatorNotAvailable",
176+
Message: "Cluster operator test-co is still updating",
177+
Name: "test-co",
172178
},
173179
}, {
174180
name: "cluster operator reporting desired operator and old versions for 2 operands",
@@ -197,10 +203,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
197203
},
198204
},
199205
expErr: &payload.UpdateError{
200-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
201-
Reason: "ClusterOperatorNotAvailable",
202-
Message: "Cluster operator test-co is still updating",
203-
Name: "test-co",
206+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
207+
Progressing: true,
208+
Reason: "ClusterOperatorNotAvailable",
209+
Message: "Cluster operator test-co is still updating",
210+
Name: "test-co",
204211
},
205212
}, {
206213
name: "cluster operator reporting desired operator and mix of old and desired versions for 2 operands",
@@ -229,10 +236,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
229236
},
230237
},
231238
expErr: &payload.UpdateError{
232-
Nested: fmt.Errorf("cluster operator test-co is still updating"),
233-
Reason: "ClusterOperatorNotAvailable",
234-
Message: "Cluster operator test-co is still updating",
235-
Name: "test-co",
239+
Nested: fmt.Errorf("cluster operator test-co is still updating"),
240+
Progressing: true,
241+
Reason: "ClusterOperatorNotAvailable",
242+
Message: "Cluster operator test-co is still updating",
243+
Name: "test-co",
236244
},
237245
}, {
238246
name: "cluster operator reporting desired versions and no conditions",
@@ -257,10 +265,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
257265
},
258266
},
259267
expErr: &payload.UpdateError{
260-
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=false, progressing=true, degraded=true"),
261-
Reason: "ClusterOperatorNotAvailable",
262-
Message: "Cluster operator test-co has not yet reported success",
263-
Name: "test-co",
268+
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=false, progressing=true, degraded=true"),
269+
Progressing: true,
270+
Reason: "ClusterOperatorNotAvailable",
271+
Message: "Cluster operator test-co has not yet reported success",
272+
Name: "test-co",
264273
},
265274
}, {
266275
name: "cluster operator reporting progressing=true",
@@ -286,10 +295,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
286295
},
287296
},
288297
expErr: &payload.UpdateError{
289-
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=false, progressing=true, degraded=true"),
290-
Reason: "ClusterOperatorNotAvailable",
291-
Message: "Cluster operator test-co has not yet reported success",
292-
Name: "test-co",
298+
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=false, progressing=true, degraded=true"),
299+
Progressing: true,
300+
Reason: "ClusterOperatorNotAvailable",
301+
Message: "Cluster operator test-co has not yet reported success",
302+
Name: "test-co",
293303
},
294304
}, {
295305
name: "cluster operator reporting degraded=true",
@@ -315,10 +325,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
315325
},
316326
},
317327
expErr: &payload.UpdateError{
318-
Nested: fmt.Errorf("cluster operator test-co is reporting a failure: random error"),
319-
Reason: "ClusterOperatorDegraded",
320-
Message: "Cluster operator test-co is reporting a failure: random error",
321-
Name: "test-co",
328+
Nested: fmt.Errorf("cluster operator test-co is reporting a failure: random error"),
329+
Progressing: false,
330+
Reason: "ClusterOperatorDegraded",
331+
Message: "Cluster operator test-co is reporting a failure: random error",
332+
Name: "test-co",
322333
},
323334
}, {
324335
name: "cluster operator reporting available=true progressing=true",
@@ -344,10 +355,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
344355
},
345356
},
346357
expErr: &payload.UpdateError{
347-
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=true, progressing=true, degraded=true"),
348-
Reason: "ClusterOperatorNotAvailable",
349-
Message: "Cluster operator test-co has not yet reported success",
350-
Name: "test-co",
358+
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=true, progressing=true, degraded=true"),
359+
Progressing: true,
360+
Reason: "ClusterOperatorNotAvailable",
361+
Message: "Cluster operator test-co has not yet reported success",
362+
Name: "test-co",
351363
},
352364
}, {
353365
name: "cluster operator reporting available=true degraded=true",
@@ -373,10 +385,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
373385
},
374386
},
375387
expErr: &payload.UpdateError{
376-
Nested: fmt.Errorf("cluster operator test-co is reporting a failure: random error"),
377-
Reason: "ClusterOperatorDegraded",
378-
Message: "Cluster operator test-co is reporting a failure: random error",
379-
Name: "test-co",
388+
Nested: fmt.Errorf("cluster operator test-co is reporting a failure: random error"),
389+
Progressing: false,
390+
Reason: "ClusterOperatorDegraded",
391+
Message: "Cluster operator test-co is reporting a failure: random error",
392+
Name: "test-co",
380393
},
381394
}, {
382395
name: "cluster operator reporting available=true progressing=true degraded=true",
@@ -402,10 +415,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
402415
},
403416
},
404417
expErr: &payload.UpdateError{
405-
Nested: fmt.Errorf("cluster operator test-co is reporting a failure: random error"),
406-
Reason: "ClusterOperatorDegraded",
407-
Message: "Cluster operator test-co is reporting a failure: random error",
408-
Name: "test-co",
418+
Nested: fmt.Errorf("cluster operator test-co is reporting a failure: random error"),
419+
Progressing: false,
420+
Reason: "ClusterOperatorDegraded",
421+
Message: "Cluster operator test-co is reporting a failure: random error",
422+
Name: "test-co",
409423
},
410424
}, {
411425
name: "cluster operator reporting available=true no progressing or degraded",
@@ -431,10 +445,11 @@ func Test_waitForOperatorStatusToBeDone(t *testing.T) {
431445
},
432446
},
433447
expErr: &payload.UpdateError{
434-
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=true, progressing=true, degraded=true"),
435-
Reason: "ClusterOperatorNotAvailable",
436-
Message: "Cluster operator test-co has not yet reported success",
437-
Name: "test-co",
448+
Nested: fmt.Errorf("cluster operator test-co is not done; it is available=true, progressing=true, degraded=true"),
449+
Progressing: true,
450+
Reason: "ClusterOperatorNotAvailable",
451+
Message: "Cluster operator test-co has not yet reported success",
452+
Name: "test-co",
438453
},
439454
}, {
440455
name: "cluster operator reporting available=true progressing=false degraded=false",

‎pkg/cvo/status.go

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ const ClusterVersionInvalid configv1.ClusterStatusConditionType = "Invalid"
156156
// syncStatus calculates the new status of the ClusterVersion based on the current sync state and any
157157
// validation errors found. We allow the caller to pass the original object to avoid DeepCopying twice.
158158
func (optr *Operator) syncStatus(ctx context.Context, original, config *configv1.ClusterVersion, status *SyncWorkerStatus, validationErrs field.ErrorList) error {
159-
klog.V(5).Infof("Synchronizing errs=%#v status=%#v", validationErrs, status)
159+
klog.Infof("Synchronizing errs=%#v status=%#v", validationErrs, status)
160160

161161
cvUpdated := false
162162
// update the config with the latest available updates
@@ -242,6 +242,7 @@ func (optr *Operator) syncStatus(ctx context.Context, original, config *configv1
242242
progressReason, progressShortMessage, skipFailure := convertErrorToProgressing(config.Status.History, now.Time, status)
243243

244244
if err := status.Failure; err != nil && !skipFailure {
245+
klog.Infof("!!!! NOT skipFailure")
245246
var reason string
246247
msg := "an error occurred"
247248
if uErr, ok := err.(*payload.UpdateError); ok {
@@ -278,6 +279,7 @@ func (optr *Operator) syncStatus(ctx context.Context, original, config *configv1
278279
}
279280

280281
} else {
282+
klog.Infof("!!!! skipFailure")
281283
// clear the failure condition
282284
resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: ClusterStatusFailing, Status: configv1.ConditionFalse, LastTransitionTime: now})
283285

@@ -347,19 +349,32 @@ func (optr *Operator) syncStatus(ctx context.Context, original, config *configv1
347349
// unavailable AND the general payload task making progress towards its goal. An operator is given 40 minutes since
348350
// its last update to go ready, or an hour has elapsed since the update began, before the condition is ignored.
349351
func convertErrorToProgressing(history []configv1.UpdateHistory, now time.Time, status *SyncWorkerStatus) (reason string, message string, ok bool) {
352+
uErr, ok := status.Failure.(*payload.UpdateError)
353+
if ok {
354+
klog.Infof("!!!! uErr=%#v")
355+
}
350356
if len(history) == 0 || status.Failure == nil || status.Reconciling || status.LastProgress.IsZero() {
357+
/*
358+
if ok && uErr.Progressing {
359+
return "", "", true
360+
}
361+
*/
362+
klog.Infof("!!!! 10")
351363
return "", "", false
352364
}
353365
if now.Sub(status.LastProgress) > 40*time.Minute || now.Sub(history[0].StartedTime.Time) > time.Hour {
366+
klog.Infof("!!!! 20")
354367
return "", "", false
355368
}
356-
uErr, ok := status.Failure.(*payload.UpdateError)
357369
if !ok {
370+
klog.Infof("!!!! 30")
358371
return "", "", false
359372
}
360373
if uErr.Reason == "ClusterOperatorNotAvailable" || uErr.Reason == "ClusterOperatorsNotAvailable" {
374+
klog.Infof("!!!! 40")
361375
return uErr.Reason, fmt.Sprintf("waiting on %s", uErr.Name), true
362376
}
377+
klog.Infof("!!!! 50")
363378
return "", "", false
364379
}
365380

0 commit comments

Comments
 (0)