Skip to content

Commit 1d36576

Browse files
thockinjpbetz
thockin
authored and
jpbetz
committed
Introduce versioned validation test utilitizes and add fuzz tester
This makes a bold assumption: that the errors (count and basic content) will be the same across versions. If this turns out to be untrue, this may need to get more sophisticated. It should fail obviously when we hit that edge.
1 parent 31d16ff commit 1d36576

File tree

3 files changed

+284
-0
lines changed

3 files changed

+284
-0
lines changed

‎pkg/api/testing/validation.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package testing
18+
19+
import (
20+
"bytes"
21+
"sort"
22+
"strconv"
23+
"testing"
24+
25+
k8sruntime "k8s.io/apimachinery/pkg/runtime"
26+
runtimetest "k8s.io/apimachinery/pkg/runtime/testing"
27+
"k8s.io/apimachinery/pkg/util/sets"
28+
"k8s.io/apimachinery/pkg/util/validation/field"
29+
"k8s.io/kubernetes/pkg/api/legacyscheme"
30+
)
31+
32+
// VerifyVersionedValidationEquivalence tests that all versions of an API return equivalent validation errors.
33+
func VerifyVersionedValidationEquivalence(t *testing.T, obj, old k8sruntime.Object) {
34+
t.Helper()
35+
36+
// Accumulate errors from all versioned validation, per version.
37+
all := map[string]field.ErrorList{}
38+
accumulate := func(t *testing.T, gv string, errs field.ErrorList) {
39+
all[gv] = errs
40+
}
41+
if old == nil {
42+
runtimetest.RunValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, accumulate)
43+
} else {
44+
runtimetest.RunUpdateValidationForEachVersion(t, legacyscheme.Scheme, sets.Set[string]{}, obj, old, accumulate)
45+
}
46+
47+
// Make a copy so we can modify it.
48+
other := map[string]field.ErrorList{}
49+
// Index for nicer output.
50+
keys := []string{}
51+
for k, v := range all {
52+
other[k] = v
53+
keys = append(keys, k)
54+
}
55+
sort.Strings(keys)
56+
57+
// Compare each lhs to each rhs.
58+
for _, lk := range keys {
59+
lv := all[lk]
60+
// remove lk since to prevent comparison to itself and because this
61+
// iteration will compare it to any version it has not yet been
62+
// compared to. e.g. [1, 2, 3] vs. [1, 2, 3] yields:
63+
// 1 vs. 2
64+
// 1 vs. 3
65+
// 2 vs. 3
66+
delete(other, lk)
67+
// don't compare to ourself
68+
for _, rk := range keys {
69+
rv, found := other[rk]
70+
if !found {
71+
continue // done already
72+
}
73+
if len(lv) != len(rv) {
74+
t.Errorf("different error count (%d vs. %d)\n%s: %v\n%s: %v", len(lv), len(rv), lk, fmtErrs(lv), rk, fmtErrs(rv))
75+
continue
76+
}
77+
next := false
78+
for i := range lv {
79+
if l, r := lv[i], rv[i]; l.Type != r.Type || l.Detail != r.Detail {
80+
t.Errorf("different errors\n%s: %v\n%s: %v", lk, fmtErrs(lv), rk, fmtErrs(rv))
81+
next = true
82+
break
83+
}
84+
}
85+
if next {
86+
continue
87+
}
88+
}
89+
}
90+
}
91+
92+
// helper for nicer output
93+
func fmtErrs(errs field.ErrorList) string {
94+
if len(errs) == 0 {
95+
return "<no errors>"
96+
}
97+
if len(errs) == 1 {
98+
return strconv.Quote(errs[0].Error())
99+
}
100+
buf := bytes.Buffer{}
101+
for _, e := range errs {
102+
buf.WriteString("\n")
103+
buf.WriteString(strconv.Quote(e.Error()))
104+
}
105+
106+
return buf.String()
107+
}

‎pkg/api/testing/validation_test.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package testing
18+
19+
import (
20+
"math/rand"
21+
"testing"
22+
23+
"k8s.io/apimachinery/pkg/api/apitesting/fuzzer"
24+
"k8s.io/apimachinery/pkg/api/apitesting/roundtrip"
25+
"k8s.io/apimachinery/pkg/runtime/schema"
26+
"k8s.io/kubernetes/pkg/api/legacyscheme"
27+
)
28+
29+
// FIXME: Automatically finds all group/versions supporting declarative validation, or add
30+
// a reflexive test that verifies that they are all registered.
31+
func TestVersionedValidationByFuzzing(t *testing.T) {
32+
typesWithDeclarativeValidation := []schema.GroupVersion{
33+
// Registered group versions for versioned validation fuzz testing:
34+
}
35+
36+
for _, gv := range typesWithDeclarativeValidation {
37+
t.Run(gv.String(), func(t *testing.T) {
38+
for i := 0; i < *roundtrip.FuzzIters; i++ {
39+
f := fuzzer.FuzzerFor(FuzzerFuncs, rand.NewSource(rand.Int63()), legacyscheme.Codecs)
40+
for kind := range legacyscheme.Scheme.KnownTypes(gv) {
41+
obj, err := legacyscheme.Scheme.New(gv.WithKind(kind))
42+
if err != nil {
43+
t.Fatalf("could not create a %v: %s", kind, err)
44+
}
45+
f.Fill(obj)
46+
VerifyVersionedValidationEquivalence(t, obj, nil)
47+
48+
old, err := legacyscheme.Scheme.New(gv.WithKind(kind))
49+
if err != nil {
50+
t.Fatalf("could not create a %v: %s", kind, err)
51+
}
52+
f.Fill(old)
53+
VerifyVersionedValidationEquivalence(t, obj, old)
54+
}
55+
}
56+
})
57+
}
58+
}

‎staging/src/k8s.io/apimachinery/pkg/runtime/testing/validation.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package testing
18+
19+
import (
20+
"context"
21+
"testing"
22+
23+
"k8s.io/apimachinery/pkg/runtime"
24+
"k8s.io/apimachinery/pkg/util/sets"
25+
"k8s.io/apimachinery/pkg/util/validation/field"
26+
)
27+
28+
type VersionValidationRunner func(t *testing.T, gv string, versionValidationErrors field.ErrorList)
29+
30+
// RunValidationForEachVersion runs f as a subtest of t for each version of the given unversioned object.
31+
// Each subtest is named by GroupVersionKind. Each call to f is provided the field.ErrorList results
32+
// of converting the unversioned object to a version and validating it.
33+
//
34+
// Only autogenerated validation is run. To test both handwritten and autogenerated validation:
35+
//
36+
// RunValidationForEachVersion(t, testCase.pod, func(t *testing.T, versionValidationErrors field.ErrorList) {
37+
// errs := ValidatePod(testCase.obj) // hand written validation
38+
// errs = append(errs, versionValidationErrors...) // generated declarative validation
39+
// // Validate that the errors are what was expected for this test case.
40+
// })
41+
func RunValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner) {
42+
runValidation(t, scheme, options, unversioned, fn)
43+
}
44+
45+
// RunUpdateValidationForEachVersion is like RunValidationForEachVersion but for update validation.
46+
func RunUpdateValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) {
47+
runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn)
48+
}
49+
50+
// RunStatusValidationForEachVersion is like RunUpdateValidationForEachVersion but for status validation.
51+
func RunStatusValidationForEachVersion(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned, unversionedOld runtime.Object, fn VersionValidationRunner) {
52+
runUpdateValidation(t, scheme, options, unversioned, unversionedOld, fn, "status")
53+
}
54+
55+
func runValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversioned runtime.Object, fn VersionValidationRunner, subresources ...string) {
56+
unversionedGVKs, _, err := scheme.ObjectKinds(unversioned)
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
for _, unversionedGVK := range unversionedGVKs {
61+
gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind())
62+
for _, gv := range gvs {
63+
gvk := gv.WithKind(unversionedGVK.Kind)
64+
t.Run(gvk.String(), func(t *testing.T) {
65+
if gvk.Version != runtime.APIVersionInternal { // skip internal
66+
versioned, err := scheme.New(gvk)
67+
if err != nil {
68+
t.Fatal(err)
69+
}
70+
err = scheme.Convert(unversioned, versioned, nil)
71+
if err != nil {
72+
t.Fatal(err)
73+
}
74+
fn(t, gv.String(), scheme.Validate(context.Background(), options, versioned, subresources...))
75+
}
76+
})
77+
}
78+
}
79+
}
80+
81+
func runUpdateValidation(t *testing.T, scheme *runtime.Scheme, options sets.Set[string], unversionedNew, unversionedOld runtime.Object, fn VersionValidationRunner, subresources ...string) {
82+
unversionedGVKs, _, err := scheme.ObjectKinds(unversionedNew)
83+
if err != nil {
84+
t.Fatal(err)
85+
}
86+
for _, unversionedGVK := range unversionedGVKs {
87+
gvs := scheme.VersionsForGroupKind(unversionedGVK.GroupKind())
88+
for _, gv := range gvs {
89+
gvk := gv.WithKind(unversionedGVK.Kind)
90+
t.Run(gvk.String(), func(t *testing.T) {
91+
if gvk.Version != runtime.APIVersionInternal { // skip internal
92+
versionedNew, err := scheme.New(gvk)
93+
if err != nil {
94+
t.Fatal(err)
95+
}
96+
err = scheme.Convert(unversionedNew, versionedNew, nil)
97+
if err != nil {
98+
t.Fatal(err)
99+
}
100+
101+
var versionedOld runtime.Object
102+
if unversionedOld != nil {
103+
versionedOld, err = scheme.New(gvk)
104+
if err != nil {
105+
t.Fatal(err)
106+
}
107+
108+
err = scheme.Convert(unversionedOld, versionedOld, nil)
109+
if err != nil {
110+
t.Fatal(err)
111+
}
112+
}
113+
114+
fn(t, gv.String(), scheme.ValidateUpdate(context.Background(), options, versionedNew, versionedOld, subresources...))
115+
}
116+
})
117+
}
118+
}
119+
}

0 commit comments

Comments
 (0)