Skip to content

Commit 2abeb0d

Browse files
atulv7dkoshkin
andauthored
fix(auto-cert-renewal): adds 0 as valid value for daysBeforeExpiry (#1218)
**What problem does this PR solve?**: As of now to disable CAPI based automated certificate renewal we have to set `autoRenewCertificates` to `nil` which results in SSA patch [issue](kubernetes/kubernetes#117447). To avoid this added 0 as a valid value for `autoRenewCertificates.daysBeforeExpiry` denoting the feat is disabled. **Which issue(s) this PR fixes**: Fixes [NCN-108329](https://jira.nutanix.com/browse/NCN-108329) **How Has This Been Tested?**: <!-- Please describe the tests that you ran to verify your changes. Provide output from the tests and any manual steps needed to replicate the tests. --> - Tested using the dev workflow along with the konvoy [changes](mesosphere/konvoy2#3959) --------- Co-authored-by: Dimitri Koshkin <[email protected]>
1 parent d10ab4c commit 2abeb0d

File tree

11 files changed

+286
-33
lines changed

11 files changed

+286
-33
lines changed

api/v1alpha1/controlplane_types.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@ type GenericControlPlaneSpec struct {
1313
type AutoRenewCertificatesSpec struct {
1414
// DaysBeforeExpiry indicates a rollout needs to be performed if the
1515
// certificates of the control plane will expire within the specified days.
16+
// Set to 0 to disable automated certificate renewal.
1617
// +kubebuilder:validation:Required
17-
// +kubebuilder:validation:Minimum=7
18-
DaysBeforeExpiry int32 `json:"daysBeforeExpiry,omitempty"`
18+
// +kubebuilder:validation:XValidation:rule="self == 0 || self >= 7",message="Value must be 0 or at least 7"
19+
DaysBeforeExpiry int32 `json:"daysBeforeExpiry"`
1920
}
2021

2122
// DockerControlPlaneSpec defines the desired state of the control plane for a Docker cluster.

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,9 +339,12 @@ spec:
339339
description: |-
340340
DaysBeforeExpiry indicates a rollout needs to be performed if the
341341
certificates of the control plane will expire within the specified days.
342+
Set to 0 to disable automated certificate renewal.
342343
format: int32
343-
minimum: 7
344344
type: integer
345+
x-kubernetes-validations:
346+
- message: Value must be 0 or at least 7
347+
rule: self == 0 || self >= 7
345348
required:
346349
- daysBeforeExpiry
347350
type: object

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,9 +304,12 @@ spec:
304304
description: |-
305305
DaysBeforeExpiry indicates a rollout needs to be performed if the
306306
certificates of the control plane will expire within the specified days.
307+
Set to 0 to disable automated certificate renewal.
307308
format: int32
308-
minimum: 7
309309
type: integer
310+
x-kubernetes-validations:
311+
- message: Value must be 0 or at least 7
312+
rule: self == 0 || self >= 7
310313
required:
311314
- daysBeforeExpiry
312315
type: object

api/v1alpha1/crds/caren.nutanix.com_dockerworkernodeconfigs.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,12 @@ spec:
5252
description: |-
5353
DaysBeforeExpiry indicates a rollout needs to be performed if the
5454
certificates of the control plane will expire within the specified days.
55+
Set to 0 to disable automated certificate renewal.
5556
format: int32
56-
minimum: 7
5757
type: integer
58+
x-kubernetes-validations:
59+
- message: Value must be 0 or at least 7
60+
rule: self == 0 || self >= 7
5861
required:
5962
- daysBeforeExpiry
6063
type: object

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,9 +304,12 @@ spec:
304304
description: |-
305305
DaysBeforeExpiry indicates a rollout needs to be performed if the
306306
certificates of the control plane will expire within the specified days.
307+
Set to 0 to disable automated certificate renewal.
307308
format: int32
308-
minimum: 7
309309
type: integer
310+
x-kubernetes-validations:
311+
- message: Value must be 0 or at least 7
312+
rule: self == 0 || self >= 7
310313
required:
311314
- daysBeforeExpiry
312315
type: object

common/pkg/testutils/capitest/variables.go

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,30 @@ type VariableTestDef struct {
2626
ExpectError bool
2727
}
2828

29-
func ValidateDiscoverVariables[T mutation.DiscoverVariables](
29+
func ValidateDiscoverVariables[V mutation.DiscoverVariables](
3030
t *testing.T,
3131
variableName string,
3232
variableSchema *clusterv1.VariableSchema,
3333
variableRequired bool,
34-
handlerCreator func() T,
34+
handlerCreator func() V,
35+
variableTestDefs ...VariableTestDef,
36+
) {
37+
ValidateDiscoverVariablesAs[V, any](
38+
t,
39+
variableName,
40+
variableSchema,
41+
variableRequired,
42+
handlerCreator,
43+
variableTestDefs...,
44+
)
45+
}
46+
47+
func ValidateDiscoverVariablesAs[V mutation.DiscoverVariables, T any](
48+
t *testing.T,
49+
variableName string,
50+
variableSchema *clusterv1.VariableSchema,
51+
variableRequired bool,
52+
handlerCreator func() V,
3553
variableTestDefs ...VariableTestDef,
3654
) {
3755
t.Helper()
@@ -72,7 +90,7 @@ func ValidateDiscoverVariables[T mutation.DiscoverVariables](
7290
case tt.OldVals != nil:
7391
encodedOldVals, err := json.Marshal(tt.OldVals)
7492
g.Expect(err).NotTo(gomega.HaveOccurred())
75-
validateErr = openapi.ValidateClusterVariableUpdate(
93+
validateErr = openapi.ValidateClusterVariableUpdate[T](
7694
&clusterv1.ClusterVariable{
7795
Name: variableName,
7896
Value: apiextensionsv1.JSON{Raw: encodedVals},
@@ -85,7 +103,7 @@ func ValidateDiscoverVariables[T mutation.DiscoverVariables](
85103
field.NewPath(variableName),
86104
).ToAggregate()
87105
default:
88-
validateErr = openapi.ValidateClusterVariable(
106+
validateErr = openapi.ValidateClusterVariable[T](
89107
&clusterv1.ClusterVariable{
90108
Name: variableName,
91109
Value: apiextensionsv1.JSON{Raw: encodedVals},

common/pkg/testutils/openapi/cel.go

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright 2025 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package openapi
5+
6+
import (
7+
"context"
8+
"reflect"
9+
"strings"
10+
11+
structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
12+
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel"
13+
"k8s.io/apimachinery/pkg/util/validation/field"
14+
)
15+
16+
// validateCELRecursively recursively validates CEL rules across all schema.Properties.
17+
// validator.Validate() does not traverse nested structs, so we need to do it manually.
18+
// It walks each field of the struct, checking if it has a CEL validation rule,
19+
// and if so, it validates the field using the CEL validator.
20+
func validateCELRecursively(
21+
ctx context.Context,
22+
validator *cel.Validator,
23+
path *field.Path,
24+
schema *structuralschema.Structural,
25+
newVal reflect.Value,
26+
oldVal reflect.Value,
27+
budget int64,
28+
) field.ErrorList {
29+
var errs field.ErrorList
30+
31+
// Prepare values for top-level Validate call
32+
newIface := reflectValueToInterface(newVal)
33+
oldIface := reflectValueToInterface(oldVal)
34+
35+
selfErrs, _ := validator.Validate(ctx, path, schema, newIface, oldIface, budget)
36+
errs = append(errs, selfErrs...)
37+
38+
// Dereference pointers
39+
if newVal.Kind() == reflect.Pointer && !newVal.IsNil() {
40+
newVal = newVal.Elem()
41+
}
42+
if oldVal.Kind() == reflect.Pointer && !oldVal.IsNil() {
43+
oldVal = oldVal.Elem()
44+
}
45+
46+
// Recurse only if newVal is struct
47+
if newVal.Kind() != reflect.Struct {
48+
return errs
49+
}
50+
51+
typ := newVal.Type()
52+
for i := 0; i < newVal.NumField(); i++ {
53+
fieldType := typ.Field(i)
54+
newFieldVal := newVal.Field(i)
55+
56+
if fieldType.PkgPath != "" {
57+
continue // unexported
58+
}
59+
60+
// Get old value safely
61+
var oldFieldVal reflect.Value
62+
if oldVal.IsValid() && oldVal.Kind() == reflect.Struct {
63+
oldFieldVal = oldVal.FieldByName(fieldType.Name)
64+
}
65+
66+
// Handle embedded fields
67+
if fieldType.Anonymous && newFieldVal.Kind() == reflect.Struct {
68+
errs = append(errs,
69+
validateCELRecursively(ctx, validator, path, schema, newFieldVal, oldFieldVal, budget)...)
70+
continue
71+
}
72+
73+
jsonName := jsonFieldName(fieldType)
74+
if jsonName == "" {
75+
continue
76+
}
77+
78+
subSchema, okSchema := schema.Properties[jsonName]
79+
subValidator, okValidator := validator.Properties[jsonName]
80+
if !okSchema || !okValidator {
81+
continue
82+
}
83+
84+
subPath := path.Child(jsonName)
85+
errs = append(errs,
86+
validateCELRecursively(ctx, &subValidator, subPath, &subSchema, newFieldVal, oldFieldVal, budget)...)
87+
}
88+
89+
return errs
90+
}
91+
92+
func reflectValueToInterface(v reflect.Value) interface{} {
93+
if !v.IsValid() {
94+
return nil
95+
}
96+
if v.Kind() == reflect.Pointer && v.IsNil() {
97+
return nil
98+
}
99+
return v.Interface()
100+
}
101+
102+
func jsonFieldName(field reflect.StructField) string {
103+
tag := field.Tag.Get("json")
104+
if tag == "-" {
105+
return ""
106+
}
107+
if tag == "" {
108+
return field.Name
109+
}
110+
return strings.Split(tag, ",")[0]
111+
}

common/pkg/testutils/openapi/validate.go

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"context"
88
"encoding/json"
99
"fmt"
10+
"reflect"
1011
"strings"
1112

1213
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
@@ -24,7 +25,7 @@ import (
2425
// See: https://github.com/kubernetes-sigs/cluster-api/blob/v1.5.1/internal/topology/variables/cluster_variable_validation.go#L118
2526
//
2627
//nolint:lll // Adding for URL above, does not work when adding to end of line in a comment block.
27-
func ValidateClusterVariable(
28+
func ValidateClusterVariable[T any](
2829
value *clusterv1.ClusterVariable,
2930
definition *clusterv1.ClusterClassVariable,
3031
fldPath *field.Path,
@@ -34,7 +35,7 @@ func ValidateClusterVariable(
3435
return field.ErrorList{err}
3536
}
3637

37-
variableValue, err := unmarshalAndDefaultVariableValue(fldPath, value, structuralSchema)
38+
variableValue, err := unmarshalAndDefaultVariableValue[T](fldPath, value, structuralSchema)
3839
if err != nil {
3940
return field.ErrorList{err}
4041
}
@@ -45,26 +46,27 @@ func ValidateClusterVariable(
4546
return err
4647
}
4748

49+
var oldVariableValue T
4850
// Validate variable against the schema using CEL.
49-
if err := validateCEL(fldPath, variableValue, nil, structuralSchema); err != nil {
51+
if err := validateCEL[T](fldPath, variableValue, oldVariableValue, structuralSchema); err != nil {
5052
return err
5153
}
5254

5355
return validateUnknownFields(fldPath, value, variableValue, apiExtensionsSchema)
5456
}
5557

56-
func unmarshalAndDefaultVariableValue(
58+
func unmarshalAndDefaultVariableValue[T any](
5759
fldPath *field.Path,
5860
value *clusterv1.ClusterVariable,
5961
s *structuralschema.Structural,
60-
) (any, *field.Error) {
62+
) (T, *field.Error) {
6163
// Parse JSON value.
62-
var variableValue any
64+
var variableValue T
6365
// Only try to unmarshal the clusterVariable if it is not nil, otherwise the variableValue is nil.
6466
// Note: A clusterVariable with a nil value is the result of setting the variable value to "null" via YAML.
6567
if value.Value.Raw != nil {
6668
if err := json.Unmarshal(value.Value.Raw, &variableValue); err != nil {
67-
return nil, field.Invalid(
69+
return variableValue, field.Invalid(
6870
fldPath.Child("value"), string(value.Value.Raw),
6971
fmt.Sprintf("variable %q could not be parsed: %v", value.Name, err),
7072
)
@@ -122,9 +124,9 @@ func validatorAndSchemas(
122124
return validator, apiExtensionsSchema, s, nil
123125
}
124126

125-
func validateCEL(
127+
func validateCEL[T any](
126128
fldPath *field.Path,
127-
variableValue, oldVariableValue any,
129+
variableValue, oldVariableValue T,
128130
structuralSchema *structuralschema.Structural,
129131
) field.ErrorList {
130132
// Note: k/k CR validation also uses celconfig.PerCallLimit when creating the validator for a custom resource.
@@ -138,12 +140,13 @@ func validateCEL(
138140

139141
// Note: k/k CRD validation also uses celconfig.RuntimeCELCostBudget for the Validate call.
140142
// The current RuntimeCELCostBudget gives roughly 1 second for the validation of a variable value.
141-
if validationErrors, _ := celValidator.Validate(
143+
if validationErrors := validateCELRecursively(
142144
context.Background(),
145+
celValidator,
143146
fldPath.Child("value"),
144147
structuralSchema,
145-
variableValue,
146-
oldVariableValue,
148+
reflect.ValueOf(variableValue),
149+
reflect.ValueOf(oldVariableValue),
147150
celconfig.RuntimeCELCostBudget,
148151
); len(validationErrors) > 0 {
149152
var allErrs field.ErrorList
@@ -215,8 +218,8 @@ func validateUnknownFields(
215218
return nil
216219
}
217220

218-
// ValidateClusterVariable validates an update to a clusterVariable.
219-
func ValidateClusterVariableUpdate(
221+
// ValidateClusterVariableUpdate validates an update to a clusterVariable.
222+
func ValidateClusterVariableUpdate[T any](
220223
value, oldValue *clusterv1.ClusterVariable,
221224
definition *clusterv1.ClusterClassVariable,
222225
fldPath *field.Path,
@@ -226,12 +229,12 @@ func ValidateClusterVariableUpdate(
226229
return field.ErrorList{err}
227230
}
228231

229-
variableValue, err := unmarshalAndDefaultVariableValue(fldPath, value, structuralSchema)
232+
variableValue, err := unmarshalAndDefaultVariableValue[T](fldPath, value, structuralSchema)
230233
if err != nil {
231234
return field.ErrorList{err}
232235
}
233236

234-
oldVariableValue, err := unmarshalAndDefaultVariableValue(fldPath, oldValue, structuralSchema)
237+
oldVariableValue, err := unmarshalAndDefaultVariableValue[T](fldPath, oldValue, structuralSchema)
235238
if err != nil {
236239
return field.ErrorList{err}
237240
}
@@ -243,7 +246,7 @@ func ValidateClusterVariableUpdate(
243246
}
244247

245248
// Validate variable against the schema using CEL.
246-
if err := validateCEL(fldPath, variableValue, oldVariableValue, structuralSchema); err != nil {
249+
if err := validateCEL[T](fldPath, variableValue, oldVariableValue, structuralSchema); err != nil {
247250
return err
248251
}
249252

pkg/handlers/generic/mutation/autorenewcerts/inject.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,9 @@ func (h *autoRenewCerts) Mutate(
7070
if err != nil {
7171
if variables.IsNotFoundError(err) {
7272
log.V(5).Info("Control Plane auto renew certs variable not defined")
73-
return nil
73+
} else {
74+
return err
7475
}
75-
76-
return err
7776
}
7877

7978
log = log.WithValues(
@@ -92,10 +91,18 @@ func (h *autoRenewCerts) Mutate(
9291
selectors.ControlPlane(),
9392
log,
9493
func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
95-
log.WithValues(
94+
log = log.WithValues(
9695
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
9796
"patchedObjectName", client.ObjectKeyFromObject(obj),
98-
).Info(fmt.Sprintf(
97+
)
98+
99+
if autoRenewCertsVar.DaysBeforeExpiry == 0 {
100+
log.Info("removing auto renew certs config from control plane kubeadm config spec")
101+
obj.Spec.Template.Spec.RolloutBefore = nil
102+
return nil
103+
}
104+
105+
log.Info(fmt.Sprintf(
99106
"adding auto renew certs config for %d days before expiry to control plane kubeadm config spec",
100107
autoRenewCertsVar.DaysBeforeExpiry,
101108
))

0 commit comments

Comments
 (0)