Add ability to auto retry failed resources

api/v1alpha1/biossettings_types.go

@@ -17,6 +17,11 @@ type BIOSSettingsTemplate struct {
 	// +optional
 	SettingsFlow []SettingsFlowItem `json:"settingsFlow,omitempty"`
 
+	// FailedAutoRetryCount is the number of times the controller should automatically retry the BIOSSettings upgrade in case of failure before giving up.
+	// kubebuilder:validation:Minimum=0
+	// +optional
+	FailedAutoRetryCount *int32 `json:"failedAutoRetryCount,omitempty"`
+
 	// ServerMaintenancePolicy is a maintenance policy to be enforced on the server.
 	// +optional
 	ServerMaintenancePolicy ServerMaintenancePolicy `json:"serverMaintenancePolicy,omitempty"`
@@ -94,6 +99,10 @@ type BIOSSettingsStatus struct {
 	// +optional
 	LastAppliedTime *metav1.Time `json:"lastAppliedTime,omitempty"`
 
+	// AutoRetryCountRemaining is the number of remaining times the controller will automatically retry the BIOSSettings upgrade in case of failure before giving up.
+	// +optional
+	AutoRetryCountRemaining *int32 `json:"autoRetryCountRemaining,omitempty"`
+
 	// Conditions represents the latest available observations of the BIOSSettings's current state.
 	// +patchStrategy=merge
 	// +patchMergeKey=type

api/v1alpha1/biosversion_types.go

@@ -44,6 +44,11 @@ type BIOSVersionTemplate struct {
 	// ServerMaintenancePolicy is a maintenance policy to be enforced on the server.
 	// +optional
 	ServerMaintenancePolicy ServerMaintenancePolicy `json:"serverMaintenancePolicy,omitempty"`
+
+	// FailedAutoRetryCount is the number of times the controller should automatically retry the BIOSVersion upgrade in case of failure before giving up.
+	// kubebuilder:validation:Minimum=0
+	// +optional
+	FailedAutoRetryCount *int32 `json:"failedAutoRetryCount,omitempty"`
 }
 
 // BIOSVersionSpec defines the desired state of BIOSVersion.
@@ -85,6 +90,10 @@ type BIOSVersionStatus struct {
 	// +optional
 	UpgradeTask *Task `json:"upgradeTask,omitempty"`
 
+	// AutoRetryCountRemaining is the number of remaining times the controller will automatically retry the BIOSVersion upgrade in case of failure before giving up.
+	// +optional
+	AutoRetryCountRemaining *int32 `json:"autoRetryCountRemaining,omitempty"`
+
 	// Conditions represents the latest available observations of the BIOS version upgrade state.
 	// +patchStrategy=merge
 	// +patchMergeKey=type

api/v1alpha1/bmcsettings_types.go

@@ -19,6 +19,11 @@ type BMCSettingsTemplate struct {
 	// +optional
 	SettingsMap map[string]string `json:"settings,omitempty"`
 
+	// FailedAutoRetryCount is the number of times the controller should automatically retry the BMCSettings upgrade in case of failure before giving up.
+	// kubebuilder:validation:Minimum=0
+	// +optional
+	FailedAutoRetryCount *int32 `json:"failedAutoRetryCount,omitempty"`
+
 	// ServerMaintenancePolicy is a maintenance policy to be applied on the server.
 	// +optional
 	ServerMaintenancePolicy ServerMaintenancePolicy `json:"serverMaintenancePolicy,omitempty"`
@@ -66,6 +71,10 @@ type BMCSettingsStatus struct {
 	// +optional
 	State BMCSettingsState `json:"state,omitempty"`
 
+	// AutoRetryCountRemaining is the number of remaining times the controller will automatically retry the BMCSettings upgrade in case of failure before giving up.
+	// +optional
+	AutoRetryCountRemaining *int32 `json:"autoRetryCountRemaining,omitempty"`
+
 	// Conditions represents the latest available observations of the BMC Settings Resource state.
 	// +patchStrategy=merge
 	// +patchMergeKey=type

api/v1alpha1/bmcversion_types.go

@@ -34,6 +34,11 @@ type BMCVersionTemplate struct {
 	// +required
 	Image ImageSpec `json:"image"`
 
+	// FailedAutoRetryCount is the number of times the controller should automatically retry the BMCVersion upgrade in case of failure before giving up.
+	// kubebuilder:validation:Minimum=0
+	// +optional
+	FailedAutoRetryCount *int32 `json:"failedAutoRetryCount,omitempty"`
+
 	// ServerMaintenancePolicy is a maintenance policy to be enforced on the server managed by referred BMC.
 	// +optional
 	ServerMaintenancePolicy ServerMaintenancePolicy `json:"serverMaintenancePolicy,omitempty"`
@@ -60,6 +65,11 @@ type BMCVersionStatus struct {
 
 	// UpgradeTask contains the state of the upgrade task created by the BMC.
 	UpgradeTask *Task `json:"upgradeTask,omitempty"`
+
+	// AutoRetryCountRemaining is the number of remaining times the controller will automatically retry the BMCVersion upgrade in case of failure before giving up.
+	// +optional
+	AutoRetryCountRemaining *int32 `json:"autoRetryCountRemaining,omitempty"`
+
 	// Conditions represents the latest available observations of the BMC version upgrade state.
 	// +patchStrategy=merge
 	// +patchMergeKey=type

bmc/common/helpers.go

@@ -10,8 +10,17 @@ import (
 	"github.com/stmcginnis/gofish/schemas"
 )
 
-// todo: merge with checkBiosAttribues after #298
-func CheckAttribues(
+type InvalidBMCSettingsError struct {
+	SettingName  string
+	SettingValue any
+	Message      string
+}
+
+func (e *InvalidBMCSettingsError) Error() string {
+	return fmt.Sprintf("invalid BMC setting %s=%v: %s", e.SettingName, e.SettingValue, e.Message)
+}
+
+func CheckAttributes(
 	attrs schemas.SettingsAttributes,
 	filtered map[string]schemas.Attributes,
 ) (reset bool, err error) {
@@ -21,7 +30,12 @@ func CheckAttribues(
 	for name, value := range attrs {
 		entryAttribute, ok := filtered[name]
 		if !ok {
-			errs = append(errs, fmt.Errorf("attribute %s not found or immutable/hidden", name))
+			err := &InvalidBMCSettingsError{
+				SettingName:  name,
+				SettingValue: value,
+				Message:      "attribute not found or is immutable/hidden",
+			}
+			errs = append(errs, err)
 			continue
 		}
 		if entryAttribute.ResetRequired {
@@ -30,39 +44,36 @@ func CheckAttribues(
 		switch entryAttribute.Type {
 		case schemas.IntegerAttributeType:
 			if _, ok := value.(int); !ok {
-				errs = append(
-					errs,
-					fmt.Errorf(
-						"attribute '%s's' value '%v' has wrong type. needed '%s' for '%v'",
-						name,
-						value,
+				err := &InvalidBMCSettingsError{
+					SettingName:  name,
+					SettingValue: value,
+					Message: fmt.Sprintf("attribute value has wrong type. needed '%s'",
 						entryAttribute.Type,
-						entryAttribute,
-					))
+					),
+				}
+				errs = append(errs, err)
 			}
 		case schemas.StringAttributeType:
 			if _, ok := value.(string); !ok {
-				errs = append(
-					errs,
-					fmt.Errorf(
-						"attribute '%s's' value '%v' has wrong type. needed '%s' for '%v'",
-						name,
-						value,
+				err := &InvalidBMCSettingsError{
+					SettingName:  name,
+					SettingValue: value,
+					Message: fmt.Sprintf("attribute value has wrong type. needed '%s'",
 						entryAttribute.Type,
-						entryAttribute,
-					))
+					),
+				}
+				errs = append(errs, err)
 			}
 		case schemas.EnumerationAttributeType:
 			if _, ok := value.(string); !ok {
-				errs = append(
-					errs,
-					fmt.Errorf(
-						"attribute '%s's' value '%v' has wrong type. needed '%s' for '%v'",
-						name,
-						value,
+				err := &InvalidBMCSettingsError{
+					SettingName:  name,
+					SettingValue: value,
+					Message: fmt.Sprintf("attribute value has wrong type. needed '%s'",
 						entryAttribute.Type,
-						entryAttribute,
-					))
+					),
+				}
+				errs = append(errs, err)
 				break
 			}
 			var validEnum bool
@@ -73,18 +84,22 @@ func CheckAttribues(
 				}
 			}
 			if !validEnum {
-				errs = append(errs, fmt.Errorf("attribute %s value is unknown. needed %v", name, entryAttribute.Value))
+				err := &InvalidBMCSettingsError{
+					SettingName:  name,
+					SettingValue: value,
+					Message:      fmt.Sprintf("attributes value is unknown. Valid Attributes %v", entryAttribute.Value),
+				}
+				errs = append(errs, err)
 			}
 		default:
-			errs = append(
-				errs,
-				fmt.Errorf(
-					"attribute '%s's' value '%v' has wrong type. needed '%s' for '%v'",
-					name,
-					value,
+			err := &InvalidBMCSettingsError{
+				SettingName:  name,
+				SettingValue: value,
+				Message: fmt.Sprintf("attribute value has wrong type. needed '%s'",
 					entryAttribute.Type,
-					entryAttribute,
-				))
+				),
+			}
+			errs = append(errs, err)
 		}
 	}
 	return reset, errors.Join(errs...)