Skip to content

Commit 68f6347

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#128322 from benluddy/cbor-storage-wiring
KEP-4222: Wire CBOR CR storage behind test-only feature gate.
2 parents 5147eeb + 950ed80 commit 68f6347

File tree

2 files changed

+159
-1
lines changed

2 files changed

+159
-1
lines changed

‎staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,18 @@ import (
3030
"k8s.io/apiextensions-apiserver/pkg/apiserver"
3131
generatedopenapi "k8s.io/apiextensions-apiserver/pkg/generated/openapi"
3232
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
33+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
34+
"k8s.io/apimachinery/pkg/runtime"
35+
"k8s.io/apimachinery/pkg/runtime/serializer/cbor"
36+
"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
3337
utilerrors "k8s.io/apimachinery/pkg/util/errors"
3438
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
39+
"k8s.io/apiserver/pkg/features"
3540
genericregistry "k8s.io/apiserver/pkg/registry/generic"
3641
genericapiserver "k8s.io/apiserver/pkg/server"
3742
genericoptions "k8s.io/apiserver/pkg/server/options"
3843
storagevalue "k8s.io/apiserver/pkg/storage/value"
44+
utilfeature "k8s.io/apiserver/pkg/util/feature"
3945
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
4046
"k8s.io/apiserver/pkg/util/openapi"
4147
"k8s.io/apiserver/pkg/util/proxy"
@@ -130,8 +136,23 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err
130136
// Avoid messing with anything outside of changes to StorageConfig as that
131137
// may lead to unexpected behavior when the options are applied.
132138
func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions, resourceTransformers storagevalue.ResourceTransformers, tracker flowcontrolrequest.StorageObjectCountTracker) genericregistry.RESTOptionsGetter {
139+
ucbor := cbor.NewSerializer(unstructuredscheme.NewUnstructuredCreator(), unstructuredscheme.NewUnstructuredObjectTyper())
140+
141+
encoder := unstructured.UnstructuredJSONScheme
142+
if utilfeature.TestOnlyFeatureGate.Enabled(features.TestOnlyCBORServingAndStorage) {
143+
encoder = ucbor
144+
}
145+
133146
etcdOptionsCopy := etcdOptions
134-
etcdOptionsCopy.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
147+
etcdOptionsCopy.StorageConfig.Codec = runtime.NewCodec(
148+
encoder,
149+
// Whether the feature gate is enabled or disabled, the decoder must be able to
150+
// recognize any resources stored using the CBOR encoder.
151+
recognizer.NewDecoder(
152+
ucbor,
153+
unstructured.UnstructuredJSONScheme,
154+
),
155+
)
135156
etcdOptionsCopy.StorageConfig.StorageObjectCountTracker = tracker
136157
etcdOptionsCopy.WatchCacheSizes = nil // this control is not provided for custom resources
137158

‎staging/src/k8s.io/apiextensions-apiserver/test/integration/cbor_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@ limitations under the License.
1717
package integration
1818

1919
import (
20+
"bytes"
2021
"context"
2122
"fmt"
23+
"path"
2224
"testing"
2325

26+
"github.com/google/uuid"
27+
2428
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
2529
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
2630
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
@@ -31,6 +35,7 @@ import (
3135
"k8s.io/apimachinery/pkg/runtime/schema"
3236
"k8s.io/apimachinery/pkg/runtime/serializer"
3337
cbor "k8s.io/apimachinery/pkg/runtime/serializer/cbor/direct"
38+
"k8s.io/apimachinery/pkg/util/json"
3439
"k8s.io/apiserver/pkg/features"
3540
utilfeature "k8s.io/apiserver/pkg/util/feature"
3641
"k8s.io/client-go/dynamic"
@@ -39,6 +44,138 @@ import (
3944
featuregatetesting "k8s.io/component-base/featuregate/testing"
4045
)
4146

47+
func TestCBORStorageEnablement(t *testing.T) {
48+
crd := &apiextensionsv1.CustomResourceDefinition{
49+
ObjectMeta: metav1.ObjectMeta{Name: "bars.mygroup.example.com"},
50+
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
51+
Group: "mygroup.example.com",
52+
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{
53+
Name: "v1beta1",
54+
Served: true,
55+
Storage: true,
56+
Schema: fixtures.AllowAllSchema(),
57+
}},
58+
Names: apiextensionsv1.CustomResourceDefinitionNames{
59+
Plural: "bars",
60+
Singular: "bar",
61+
Kind: "Bar",
62+
ListKind: "BarList",
63+
},
64+
Scope: apiextensionsv1.ClusterScoped,
65+
},
66+
}
67+
68+
etcdPrefix := uuid.New().String()
69+
70+
func() {
71+
t.Log("starting server with feature gate disabled")
72+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.TestOnlyFeatureGate, features.TestOnlyCBORServingAndStorage, false)
73+
tearDown, apiExtensionsClientset, dynamicClient, etcdClient, _, err := fixtures.StartDefaultServerWithClientsAndEtcd(t, "--etcd-prefix", etcdPrefix)
74+
if err != nil {
75+
t.Fatal(err)
76+
}
77+
defer tearDown()
78+
79+
if _, err := fixtures.CreateNewV1CustomResourceDefinition(crd, apiExtensionsClientset, dynamicClient); err != nil {
80+
t.Fatal(err)
81+
}
82+
83+
if _, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "mygroup.example.com", Version: "v1beta1", Resource: "bars"}).Create(
84+
context.TODO(),
85+
&unstructured.Unstructured{
86+
Object: map[string]interface{}{
87+
"apiVersion": "mygroup.example.com/v1beta1",
88+
"kind": "Bar",
89+
"metadata": map[string]interface{}{
90+
"name": "test-storage-json",
91+
},
92+
}},
93+
metav1.CreateOptions{},
94+
); err != nil {
95+
t.Fatal(err)
96+
}
97+
98+
response, err := etcdClient.KV.Get(context.TODO(), path.Join("/", etcdPrefix, crd.Spec.Group, crd.Spec.Names.Plural, "test-storage-json"))
99+
if err != nil {
100+
t.Fatal(err)
101+
}
102+
if n := len(response.Kvs); n != 1 {
103+
t.Fatalf("expected 1 kv, got %d", n)
104+
}
105+
if err := json.Unmarshal(response.Kvs[0].Value, new(interface{})); err != nil {
106+
t.Fatalf("failed to decode stored custom resource as json: %v", err)
107+
}
108+
}()
109+
110+
func() {
111+
t.Log("starting server with feature gate enabled")
112+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.TestOnlyFeatureGate, features.TestOnlyCBORServingAndStorage, true)
113+
tearDown, _, dynamicClient, etcdClient, _, err := fixtures.StartDefaultServerWithClientsAndEtcd(t, "--etcd-prefix", etcdPrefix)
114+
if err != nil {
115+
t.Fatal(err)
116+
}
117+
defer tearDown()
118+
119+
if _, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "mygroup.example.com", Version: "v1beta1", Resource: "bars"}).Create(
120+
context.TODO(),
121+
&unstructured.Unstructured{
122+
Object: map[string]interface{}{
123+
"apiVersion": "mygroup.example.com/v1beta1",
124+
"kind": "Bar",
125+
"metadata": map[string]interface{}{
126+
"name": "test-storage-cbor",
127+
},
128+
}},
129+
metav1.CreateOptions{},
130+
); err != nil {
131+
t.Fatal(err)
132+
}
133+
134+
response, err := etcdClient.KV.Get(context.TODO(), path.Join("/", etcdPrefix, crd.Spec.Group, crd.Spec.Names.Plural, "test-storage-cbor"))
135+
if err != nil {
136+
t.Fatal(err)
137+
}
138+
if n := len(response.Kvs); n != 1 {
139+
t.Fatalf("expected 1 kv, got %d", n)
140+
}
141+
if !bytes.HasPrefix(response.Kvs[0].Value, []byte{0xd9, 0xd9, 0xf7}) {
142+
// Check for the encoding of the "self-described CBOR" tag which acts as a
143+
// "magic number" for distinguishing CBOR from JSON. Valid CBOR data items
144+
// do not require this prefix, but the Kubernetes CBOR serializer guarantees
145+
// it.
146+
t.Fatalf(`stored custom resource lacks required "self-described CBOR" tag (prefix 0x%x)`, response.Kvs[0].Value[:3])
147+
}
148+
if err := cbor.Unmarshal(response.Kvs[0].Value, new(interface{})); err != nil {
149+
t.Fatalf("failed to decode stored custom resource as cbor: %v", err)
150+
}
151+
152+
for _, name := range []string{"test-storage-json", "test-storage-cbor"} {
153+
_, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "mygroup.example.com", Version: "v1beta1", Resource: "bars"}).Get(context.TODO(), name, metav1.GetOptions{})
154+
if err != nil {
155+
t.Errorf("failed to get cr %q: %v", name, err)
156+
}
157+
}
158+
}()
159+
160+
func() {
161+
t.Log("starting server with feature gate disabled")
162+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.TestOnlyFeatureGate, features.TestOnlyCBORServingAndStorage, false)
163+
tearDown, _, dynamicClient, _, _, err := fixtures.StartDefaultServerWithClientsAndEtcd(t, "--etcd-prefix", etcdPrefix)
164+
if err != nil {
165+
t.Fatal(err)
166+
}
167+
defer tearDown()
168+
169+
for _, name := range []string{"test-storage-json", "test-storage-cbor"} {
170+
_, err := dynamicClient.Resource(schema.GroupVersionResource{Group: "mygroup.example.com", Version: "v1beta1", Resource: "bars"}).Get(context.TODO(), name, metav1.GetOptions{})
171+
if err != nil {
172+
t.Errorf("failed to get cr %q: %v", name, err)
173+
}
174+
}
175+
}()
176+
177+
}
178+
42179
func TestCBORServingEnablement(t *testing.T) {
43180
for _, tc := range []struct {
44181
name string

0 commit comments

Comments
 (0)