Skip to content

Commit 1c82b51

Browse files
britaniarbritaniar
authored
feat: determine failure type when possible for update run (kubefleet-dev#621)
1 parent daf3165 commit 1c82b51

11 files changed

Lines changed: 231 additions & 83 deletions

‎pkg/controllers/updaterun/controller.go‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
109109
return runtime.Result{}, err
110110
}
111111

112+
// Track errors for metrics emission. The error is used to determine the failure type
113+
// (user_error vs internal_error) in the emitted metrics.
114+
var reconcileErr error
112115
// Emit the update run status metric based on status conditions in the updateRun.
113-
defer emitUpdateRunStatusMetric(updateRun)
116+
// Use a closure to capture reconcileErr by reference, so it reflects any updates made during reconciliation.
117+
defer func() { emitUpdateRunStatusMetric(updateRun, reconcileErr) }()
114118

115119
state := updateRun.GetUpdateRunSpec().State
116120

@@ -126,14 +130,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
126130
}
127131

128132
// Initialize the updateRun.
129-
var initErr error
130-
if toBeUpdatedBindings, toBeDeletedBindings, initErr = r.initialize(ctx, updateRun); initErr != nil {
131-
klog.ErrorS(initErr, "Failed to initialize the updateRun", "updateRun", runObjRef)
133+
if toBeUpdatedBindings, toBeDeletedBindings, reconcileErr = r.initialize(ctx, updateRun); reconcileErr != nil {
134+
klog.ErrorS(reconcileErr, "Failed to initialize the updateRun", "updateRun", runObjRef)
132135
// errStagedUpdatedAborted cannot be retried.
133-
if errors.Is(initErr, errStagedUpdatedAborted) {
134-
return runtime.Result{}, r.recordInitializationFailed(ctx, updateRun, initErr.Error())
136+
if errors.Is(reconcileErr, errStagedUpdatedAborted) {
137+
return runtime.Result{}, r.recordInitializationFailed(ctx, updateRun, reconcileErr.Error())
135138
}
136-
return runtime.Result{}, initErr
139+
return runtime.Result{}, reconcileErr
137140
}
138141
updatingStageIndex = 0 // start from the first stage (typically for Initialize or Run states).
139142
klog.V(2).InfoS("Initialized the updateRun", "state", state, "updateRun", runObjRef)
@@ -145,14 +148,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
145148
klog.V(2).InfoS("The updateRun is finished", "finishedSuccessfully", finishedCond.Status, "updateRun", runObjRef)
146149
return runtime.Result{}, nil
147150
}
148-
var validateErr error
149151
// Validate the updateRun status to ensure the update can be continued and get the updating stage index and cluster indices.
150-
if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, validateErr = r.validate(ctx, updateRun); validateErr != nil {
152+
if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, reconcileErr = r.validate(ctx, updateRun); reconcileErr != nil {
151153
// errStagedUpdatedAborted cannot be retried.
152-
if errors.Is(validateErr, errStagedUpdatedAborted) {
153-
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, validateErr.Error())
154+
if errors.Is(reconcileErr, errStagedUpdatedAborted) {
155+
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
154156
}
155-
return runtime.Result{}, validateErr
157+
return runtime.Result{}, reconcileErr
156158
}
157159
klog.V(2).InfoS("The updateRun is validated", "updateRun", runObjRef)
158160
}
@@ -163,45 +165,48 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
163165
return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun)
164166
}
165167

168+
var finished bool
169+
var waitTime time.Duration
166170
switch state {
167171
case placementv1beta1.StateInitialize:
168172
klog.V(2).InfoS("The updateRun is initialized but not executed, waiting to execute", "state", state, "updateRun", runObjRef)
169173
case placementv1beta1.StateRun:
170174
// Execute the updateRun.
171175
klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef)
172-
finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
173-
if errors.Is(execErr, errStagedUpdatedAborted) {
176+
finished, waitTime, reconcileErr = r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
177+
if errors.Is(reconcileErr, errStagedUpdatedAborted) {
174178
// errStagedUpdatedAborted cannot be retried.
175-
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error())
179+
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
176180
}
177181

178182
if finished {
179183
klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef)
180184
return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun)
181185
}
182186

183-
return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, execErr, state, runObjRef)
187+
return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, reconcileErr, state, runObjRef)
184188
case placementv1beta1.StateStop:
185189
// Stop the updateRun.
186190
klog.V(2).InfoS("Stopping the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef)
187-
finished, waitTime, stopErr := r.stop(updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
188-
if errors.Is(stopErr, errStagedUpdatedAborted) {
191+
finished, waitTime, reconcileErr = r.stop(updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
192+
if errors.Is(reconcileErr, errStagedUpdatedAborted) {
189193
// errStagedUpdatedAborted cannot be retried.
190-
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, stopErr.Error())
194+
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
191195
}
192196

193197
if finished {
194198
klog.V(2).InfoS("The updateRun is stopped", "updateRun", runObjRef)
195199
return runtime.Result{}, r.recordUpdateRunStopped(ctx, updateRun)
196200
}
197201

198-
return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, stopErr, state, runObjRef)
202+
return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, reconcileErr, state, runObjRef)
199203

200204
default:
201205
// Initialize, Run, or Stop are the only supported states.
202-
unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported updateRun state: %s", state))
203-
klog.ErrorS(unexpectedErr, "Invalid updateRun state", "state", state, "updateRun", runObjRef)
204-
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, unexpectedErr.Error())
206+
reconcileErr = controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported updateRun state: %s", state))
207+
klog.ErrorS(reconcileErr, "Invalid updateRun state", "state", state, "updateRun", runObjRef)
208+
// This is an internal error - unsupported state should not happen
209+
return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
205210
}
206211
return runtime.Result{}, nil
207212
}

‎pkg/controllers/updaterun/controller_integration_test.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ func generateApprovalStageTaskMetric(
377377
// the current updateRun state if the updateRun has transitioned since then.
378378
func generateMetricsLabels(
379379
updateRun *placementv1beta1.ClusterStagedUpdateRun,
380-
state, condition, status, reason string,
380+
state, condition, status, reason, failureType string,
381381
) []*prometheusclientmodel.LabelPair {
382382
return []*prometheusclientmodel.LabelPair{
383383
{Name: ptr.To("namespace"), Value: &updateRun.Namespace},
@@ -386,23 +386,24 @@ func generateMetricsLabels(
386386
{Name: ptr.To("condition"), Value: ptr.To(condition)},
387387
{Name: ptr.To("status"), Value: ptr.To(status)},
388388
{Name: ptr.To("reason"), Value: ptr.To(reason)},
389+
{Name: ptr.To("failureType"), Value: ptr.To(failureType)},
389390
}
390391
}
391392

392393
func generateInitializationSucceededMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
393394
return &prometheusclientmodel.Metric{
394395
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionInitialized),
395-
string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason),
396+
string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason, string(hubmetrics.UpdateRunFailureTypeNone)),
396397
Gauge: &prometheusclientmodel.Gauge{
397398
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
398399
},
399400
}
400401
}
401402

402-
func generateInitializationFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
403+
func generateInitializationFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun, failureType string) *prometheusclientmodel.Metric {
403404
return &prometheusclientmodel.Metric{
404405
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionInitialized),
405-
string(metav1.ConditionFalse), condition.UpdateRunInitializeFailedReason),
406+
string(metav1.ConditionFalse), condition.UpdateRunInitializeFailedReason, failureType),
406407
Gauge: &prometheusclientmodel.Gauge{
407408
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
408409
},
@@ -412,7 +413,7 @@ func generateInitializationFailedMetric(state placementv1beta1.State, updateRun
412413
func generateProgressingMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
413414
return &prometheusclientmodel.Metric{
414415
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
415-
string(metav1.ConditionTrue), condition.UpdateRunProgressingReason),
416+
string(metav1.ConditionTrue), condition.UpdateRunProgressingReason, string(hubmetrics.UpdateRunFailureTypeNone)),
416417
Gauge: &prometheusclientmodel.Gauge{
417418
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
418419
},
@@ -422,7 +423,7 @@ func generateProgressingMetric(state placementv1beta1.State, updateRun *placemen
422423
func generateWaitingMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
423424
return &prometheusclientmodel.Metric{
424425
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
425-
string(metav1.ConditionFalse), condition.UpdateRunWaitingReason),
426+
string(metav1.ConditionFalse), condition.UpdateRunWaitingReason, string(hubmetrics.UpdateRunFailureTypeNone)),
426427
Gauge: &prometheusclientmodel.Gauge{
427428
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
428429
},
@@ -432,17 +433,17 @@ func generateWaitingMetric(state placementv1beta1.State, updateRun *placementv1b
432433
func generateStuckMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
433434
return &prometheusclientmodel.Metric{
434435
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
435-
string(metav1.ConditionFalse), condition.UpdateRunStuckReason),
436+
string(metav1.ConditionFalse), condition.UpdateRunStuckReason, string(hubmetrics.UpdateRunFailureTypeInternalError)),
436437
Gauge: &prometheusclientmodel.Gauge{
437438
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
438439
},
439440
}
440441
}
441442

442-
func generateFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
443+
func generateFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun, failureType string) *prometheusclientmodel.Metric {
443444
return &prometheusclientmodel.Metric{
444445
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionSucceeded),
445-
string(metav1.ConditionFalse), condition.UpdateRunFailedReason),
446+
string(metav1.ConditionFalse), condition.UpdateRunFailedReason, failureType),
446447
Gauge: &prometheusclientmodel.Gauge{
447448
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
448449
},
@@ -452,7 +453,7 @@ func generateFailedMetric(state placementv1beta1.State, updateRun *placementv1be
452453
func generateStoppingMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
453454
return &prometheusclientmodel.Metric{
454455
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
455-
string(metav1.ConditionUnknown), condition.UpdateRunStoppingReason),
456+
string(metav1.ConditionUnknown), condition.UpdateRunStoppingReason, string(hubmetrics.UpdateRunFailureTypeNone)),
456457
Gauge: &prometheusclientmodel.Gauge{
457458
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
458459
},
@@ -462,7 +463,7 @@ func generateStoppingMetric(state placementv1beta1.State, updateRun *placementv1
462463
func generateStoppedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
463464
return &prometheusclientmodel.Metric{
464465
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
465-
string(metav1.ConditionFalse), condition.UpdateRunStoppedReason),
466+
string(metav1.ConditionFalse), condition.UpdateRunStoppedReason, string(hubmetrics.UpdateRunFailureTypeNone)),
466467
Gauge: &prometheusclientmodel.Gauge{
467468
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
468469
},
@@ -472,7 +473,7 @@ func generateStoppedMetric(state placementv1beta1.State, updateRun *placementv1b
472473
func generateSucceededMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
473474
return &prometheusclientmodel.Metric{
474475
Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionSucceeded),
475-
string(metav1.ConditionTrue), condition.UpdateRunSucceededReason),
476+
string(metav1.ConditionTrue), condition.UpdateRunSucceededReason, string(hubmetrics.UpdateRunFailureTypeNone)),
476477
Gauge: &prometheusclientmodel.Gauge{
477478
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
478479
},

‎pkg/controllers/updaterun/execution.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ func (r *Reconciler) executeUpdatingStage(
256256
"bindingSpecInSync", inSync, "bindingState", bindingSpec.State,
257257
"bindingRolloutStarted", rolloutStarted, "binding", klog.KObj(binding), "updateRun", updateRunRef)
258258
markClusterUpdatingFailed(clusterStatus, updateRun.GetGeneration(), preemptedErr.Error())
259-
clusterUpdateErrors = append(clusterUpdateErrors, fmt.Errorf("%w: %s", errStagedUpdatedAborted, preemptedErr.Error()))
259+
clusterUpdateErrors = append(clusterUpdateErrors, fmt.Errorf("%w: %w", errStagedUpdatedAborted, preemptedErr))
260260
continue
261261
}
262262

‎pkg/controllers/updaterun/execution_integration_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131

3232
clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1"
3333
placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
34+
hubmetrics "github.com/kubefleet-dev/kubefleet/pkg/metrics/hub"
3435
"github.com/kubefleet-dev/kubefleet/pkg/utils"
3536
"github.com/kubefleet-dev/kubefleet/pkg/utils/condition"
3637
)
@@ -718,7 +719,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() {
718719
validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "")
719720

720721
By("Checking update run status metrics are emitted")
721-
validateUpdateRunMetricsEmitted(generateWaitingMetric(placementv1beta1.StateRun, updateRun), generateProgressingMetric(placementv1beta1.StateRun, updateRun), generateStuckMetric(placementv1beta1.StateRun, updateRun), generateFailedMetric(placementv1beta1.StateRun, updateRun))
722+
validateUpdateRunMetricsEmitted(generateWaitingMetric(placementv1beta1.StateRun, updateRun), generateProgressingMetric(placementv1beta1.StateRun, updateRun), generateStuckMetric(placementv1beta1.StateRun, updateRun), generateFailedMetric(placementv1beta1.StateRun, updateRun, string(hubmetrics.UpdateRunFailureTypeUserError)))
722723
})
723724
})
724725

0 commit comments

Comments
 (0)