Skip to content

Commit c80f70c

Browse files
alebedev87alebedev87
committed
NE-1969: Set Degraded=True if unmanaged Gateway API CRDs exist
Set the ingress cluster operator’s `Degraded` status to `True` if unmanaged Gateway API CRDs exist on the cluster. An unmanaged Gateway API CRD is one with "gateway.networking.k8s.io" or "gateway.networking.x-k8s.io" in its `spec.group` field, and not managed by the gatewayapi controller. This commit uses the cluster operator’s `status.extension` field to store the names of unmanaged CRDs. The gatewayapi controller writes to this field, while the status controller reads it and updates the ingress cluster operator’s `Degraded` status accordingly.
1 parent 288f96c commit c80f70c

File tree

9 files changed

+403
-46
lines changed

9 files changed

+403
-46
lines changed

‎pkg/operator/controller/gateway-service-dns/controller_test.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,13 @@ func Test_Reconcile(t *testing.T) {
254254
WithScheme(scheme).
255255
WithRuntimeObjects(tc.existingObjects...).
256256
Build()
257-
cl := &testutil.FakeClientRecorder{fakeClient, t, []client.Object{}, []client.Object{}, []client.Object{}}
257+
cl := &testutil.FakeClientRecorder{
258+
Client: fakeClient,
259+
T: t,
260+
Added: []client.Object{},
261+
Updated: []client.Object{},
262+
Deleted: []client.Object{},
263+
}
258264
informer := informertest.FakeInformers{Scheme: scheme}
259265
cache := testutil.FakeCache{Informers: &informer, Reader: cl}
260266
reconciler := &reconciler{

‎pkg/operator/controller/gatewayapi/controller.go‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package gatewayapi
22

33
import (
44
"context"
5+
"fmt"
56
"sync"
67

78
logf "github.com/openshift/cluster-ingress-operator/pkg/log"
@@ -14,6 +15,7 @@ import (
1415
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1516
"k8s.io/apimachinery/pkg/types"
1617

18+
"sigs.k8s.io/controller-runtime/pkg/cache"
1719
"sigs.k8s.io/controller-runtime/pkg/client"
1820
"sigs.k8s.io/controller-runtime/pkg/controller"
1921
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -25,7 +27,10 @@ import (
2527
)
2628

2729
const (
28-
controllerName = "gatewayapi_controller"
30+
controllerName = "gatewayapi_controller"
31+
experimantalGatewayAPIGroupName = "gateway.networking.x-k8s.io"
32+
crdAPIGroupIndexFieldName = "crdAPIGroup"
33+
gatewayCRDAPIGroupIndexFieldValue = "gateway"
2934
)
3035

3136
var log = logf.Logger.WithName(controllerName)
@@ -36,6 +41,7 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) {
3641
operatorCache := mgr.GetCache()
3742
reconciler := &reconciler{
3843
client: mgr.GetClient(),
44+
cache: operatorCache,
3945
config: config,
4046
}
4147
c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: reconciler})
@@ -60,14 +66,33 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) {
6066
}}
6167
}
6268

63-
// watch for CRDs
64-
crdPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
65-
return o.(*apiextensionsv1.CustomResourceDefinition).Spec.Group == gatewayapiv1.GroupName
66-
})
69+
isGatewayAPICRD := func(o client.Object) bool {
70+
crd := o.(*apiextensionsv1.CustomResourceDefinition)
71+
return crd.Spec.Group == gatewayapiv1.GroupName || crd.Spec.Group == experimantalGatewayAPIGroupName
72+
}
73+
crdPredicate := predicate.NewPredicateFuncs(isGatewayAPICRD)
6774

75+
// watch for CRDs
6876
if err := c.Watch(source.Kind[client.Object](operatorCache, &apiextensionsv1.CustomResourceDefinition{}, handler.EnqueueRequestsFromMapFunc(toFeatureGate), crdPredicate)); err != nil {
6977
return nil, err
7078
}
79+
80+
// Index Gateway API CRDs by the "spec.group" field
81+
// to enable efficient filtering during list operations.
82+
// Currently, only Gateway API groups have a dedicated index field.
83+
if err := mgr.GetFieldIndexer().IndexField(
84+
context.Background(),
85+
&apiextensionsv1.CustomResourceDefinition{},
86+
crdAPIGroupIndexFieldName,
87+
client.IndexerFunc(func(o client.Object) []string {
88+
if isGatewayAPICRD(o) {
89+
return []string{gatewayCRDAPIGroupIndexFieldValue}
90+
}
91+
return []string{}
92+
})); err != nil {
93+
return nil, fmt.Errorf("failed to create index for custom resource definitions: %w", err)
94+
}
95+
7196
return c, nil
7297
}
7398

@@ -90,6 +115,7 @@ type reconciler struct {
90115
config Config
91116

92117
client client.Client
118+
cache cache.Cache
93119
recorder record.EventRecorder
94120
startControllers sync.Once
95121
}
@@ -107,6 +133,12 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (
107133
return reconcile.Result{}, err
108134
}
109135

136+
if crdNames, err := r.listUnmanagedGatewayAPICRDs(ctx); err != nil {
137+
return reconcile.Result{}, fmt.Errorf("failed to list unmanaged gateway CRDs: %w", err)
138+
} else if err = r.setUnmanagedGatewayAPICRDNamesStatus(ctx, crdNames); err != nil {
139+
return reconcile.Result{}, fmt.Errorf("failed to update the ingress cluster operator status: %w", err)
140+
}
141+
110142
if !r.config.GatewayAPIControllerEnabled {
111143
return reconcile.Result{}, nil
112144
}

‎pkg/operator/controller/gatewayapi/controller_test.go‎

Lines changed: 115 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package gatewayapi
22

33
import (
44
"context"
5+
"strings"
56
"testing"
67
"time"
78

@@ -16,6 +17,7 @@ import (
1617
"k8s.io/apimachinery/pkg/runtime"
1718
"k8s.io/apimachinery/pkg/types"
1819

20+
"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
1921
"sigs.k8s.io/controller-runtime/pkg/client"
2022
"sigs.k8s.io/controller-runtime/pkg/client/fake"
2123
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -30,28 +32,41 @@ func Test_Reconcile(t *testing.T) {
3032
ObjectMeta: metav1.ObjectMeta{Name: name},
3133
}
3234
}
35+
co := func(name string) *configv1.ClusterOperator {
36+
return &configv1.ClusterOperator{
37+
ObjectMeta: metav1.ObjectMeta{Name: name},
38+
}
39+
}
3340
tests := []struct {
3441
name string
3542
gatewayAPIEnabled bool
3643
gatewayAPIControllerEnabled bool
3744
existingObjects []runtime.Object
45+
existingStatusSubresource []client.Object
3846
expectCreate []client.Object
3947
expectUpdate []client.Object
4048
expectDelete []client.Object
49+
expectStatusUpdate []client.Object
4150
expectStartCtrl bool
4251
}{
4352
{
4453
name: "gateway API disabled",
4554
gatewayAPIEnabled: false,
46-
expectCreate: []client.Object{},
47-
expectUpdate: []client.Object{},
48-
expectDelete: []client.Object{},
49-
expectStartCtrl: false,
55+
existingObjects: []runtime.Object{
56+
co("ingress"),
57+
},
58+
expectCreate: []client.Object{},
59+
expectUpdate: []client.Object{},
60+
expectDelete: []client.Object{},
61+
expectStartCtrl: false,
5062
},
5163
{
5264
name: "gateway API enabled",
5365
gatewayAPIEnabled: true,
5466
gatewayAPIControllerEnabled: true,
67+
existingObjects: []runtime.Object{
68+
co("ingress"),
69+
},
5570
expectCreate: []client.Object{
5671
crd("gatewayclasses.gateway.networking.k8s.io"),
5772
crd("gateways.gateway.networking.k8s.io"),
@@ -67,6 +82,9 @@ func Test_Reconcile(t *testing.T) {
6782
name: "gateway API enabled, gateway API controller disabled",
6883
gatewayAPIEnabled: true,
6984
gatewayAPIControllerEnabled: false,
85+
existingObjects: []runtime.Object{
86+
co("ingress"),
87+
},
7088
expectCreate: []client.Object{
7189
crd("gatewayclasses.gateway.networking.k8s.io"),
7290
crd("gateways.gateway.networking.k8s.io"),
@@ -78,6 +96,55 @@ func Test_Reconcile(t *testing.T) {
7896
expectDelete: []client.Object{},
7997
expectStartCtrl: false,
8098
},
99+
{
100+
name: "unmanaged gateway API CRDs",
101+
gatewayAPIEnabled: true,
102+
gatewayAPIControllerEnabled: true,
103+
existingObjects: []runtime.Object{
104+
co("ingress"),
105+
crd("listenersets.gateway.networking.x-k8s.io"),
106+
},
107+
existingStatusSubresource: []client.Object{
108+
co("ingress"),
109+
},
110+
expectCreate: []client.Object{
111+
crd("gatewayclasses.gateway.networking.k8s.io"),
112+
crd("gateways.gateway.networking.k8s.io"),
113+
crd("grpcroutes.gateway.networking.k8s.io"),
114+
crd("httproutes.gateway.networking.k8s.io"),
115+
crd("referencegrants.gateway.networking.k8s.io"),
116+
},
117+
expectUpdate: []client.Object{},
118+
expectDelete: []client.Object{},
119+
expectStatusUpdate: []client.Object{
120+
co("ingress"),
121+
},
122+
expectStartCtrl: true,
123+
},
124+
{
125+
name: "third party CRDs",
126+
gatewayAPIEnabled: true,
127+
gatewayAPIControllerEnabled: true,
128+
existingObjects: []runtime.Object{
129+
co("ingress"),
130+
crd("thirdpartycrd1.openshift.io"),
131+
crd("thirdpartycrd2.openshift.io"),
132+
},
133+
existingStatusSubresource: []client.Object{
134+
co("ingress"),
135+
},
136+
expectCreate: []client.Object{
137+
crd("gatewayclasses.gateway.networking.k8s.io"),
138+
crd("gateways.gateway.networking.k8s.io"),
139+
crd("grpcroutes.gateway.networking.k8s.io"),
140+
crd("httproutes.gateway.networking.k8s.io"),
141+
crd("referencegrants.gateway.networking.k8s.io"),
142+
},
143+
expectUpdate: []client.Object{},
144+
expectDelete: []client.Object{},
145+
expectStatusUpdate: []client.Object{},
146+
expectStartCtrl: true,
147+
},
81148
}
82149

83150
scheme := runtime.NewScheme()
@@ -89,11 +156,30 @@ func Test_Reconcile(t *testing.T) {
89156
fakeClient := fake.NewClientBuilder().
90157
WithScheme(scheme).
91158
WithRuntimeObjects(tc.existingObjects...).
159+
WithStatusSubresource(tc.existingStatusSubresource...).
160+
WithIndex(&apiextensionsv1.CustomResourceDefinition{}, "crdAPIGroup", client.IndexerFunc(func(o client.Object) []string {
161+
if strings.Contains(o.GetName(), "gateway.networking") {
162+
return []string{"gateway"}
163+
}
164+
return []string{}
165+
})).
92166
Build()
93-
cl := &testutil.FakeClientRecorder{fakeClient, t, []client.Object{}, []client.Object{}, []client.Object{}}
167+
cl := &testutil.FakeClientRecorder{
168+
Client: fakeClient,
169+
T: t,
170+
Added: []client.Object{},
171+
Updated: []client.Object{},
172+
Deleted: []client.Object{},
173+
StatusWriter: &testutil.FakeStatusWriter{
174+
StatusWriter: fakeClient.Status(),
175+
},
176+
}
94177
ctrl := &testutil.FakeController{t, false, nil}
178+
informer := informertest.FakeInformers{Scheme: scheme}
179+
cache := &testutil.FakeCache{Informers: &informer, Reader: fakeClient}
95180
reconciler := &reconciler{
96181
client: cl,
182+
cache: cache,
97183
config: Config{
98184
GatewayAPIEnabled: tc.gatewayAPIEnabled,
99185
GatewayAPIControllerEnabled: tc.gatewayAPIControllerEnabled,
@@ -122,6 +208,7 @@ func Test_Reconcile(t *testing.T) {
122208
cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Annotations", "ResourceVersion"),
123209
cmpopts.IgnoreFields(metav1.TypeMeta{}, "Kind", "APIVersion"),
124210
cmpopts.IgnoreFields(apiextensionsv1.CustomResourceDefinition{}, "Spec"),
211+
cmpopts.IgnoreFields(configv1.ClusterOperator{}, "Status"),
125212
}
126213
if diff := cmp.Diff(tc.expectCreate, cl.Added, cmpOpts...); diff != "" {
127214
t.Fatalf("found diff between expected and actual creates: %s", diff)
@@ -132,6 +219,9 @@ func Test_Reconcile(t *testing.T) {
132219
if diff := cmp.Diff(tc.expectDelete, cl.Deleted, cmpOpts...); diff != "" {
133220
t.Fatalf("found diff between expected and actual deletes: %s", diff)
134221
}
222+
if diff := cmp.Diff(tc.expectStatusUpdate, cl.StatusWriter.Updated, cmpOpts...); diff != "" {
223+
t.Fatalf("found diff between expected and actual status updates: %s", diff)
224+
}
135225
})
136226
}
137227
}
@@ -140,11 +230,29 @@ func TestReconcileOnlyStartsControllerOnce(t *testing.T) {
140230
scheme := runtime.NewScheme()
141231
configv1.Install(scheme)
142232
apiextensionsv1.AddToScheme(scheme)
143-
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects().Build()
144-
cl := &testutil.FakeClientRecorder{fakeClient, t, []client.Object{}, []client.Object{}, []client.Object{}}
233+
fakeClient := fake.NewClientBuilder().
234+
WithScheme(scheme).
235+
WithRuntimeObjects(
236+
&configv1.ClusterOperator{
237+
ObjectMeta: metav1.ObjectMeta{Name: "ingress"},
238+
}).
239+
WithIndex(&apiextensionsv1.CustomResourceDefinition{}, "crdAPIGroup", client.IndexerFunc(func(o client.Object) []string {
240+
return []string{"gateway"} // all crds are gateway api ones
241+
})).
242+
Build()
243+
cl := &testutil.FakeClientRecorder{
244+
Client: fakeClient,
245+
T: t,
246+
Added: []client.Object{},
247+
Updated: []client.Object{},
248+
Deleted: []client.Object{},
249+
}
145250
ctrl := &testutil.FakeController{t, false, make(chan struct{})}
251+
informer := informertest.FakeInformers{Scheme: scheme}
252+
cache := &testutil.FakeCache{Informers: &informer, Reader: fakeClient}
146253
reconciler := &reconciler{
147254
client: cl,
255+
cache: cache,
148256
config: Config{
149257
GatewayAPIEnabled: true,
150258
GatewayAPIControllerEnabled: true,

‎pkg/operator/controller/gatewayapi/crds.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"k8s.io/apimachinery/pkg/api/errors"
1515
"k8s.io/apimachinery/pkg/types"
1616
utilerrors "k8s.io/apimachinery/pkg/util/errors"
17+
"sigs.k8s.io/controller-runtime/pkg/client"
1718
)
1819

1920
// managedCRDs is a list of CRDs that this controller manages.
@@ -25,6 +26,15 @@ var managedCRDs = []*apiextensionsv1.CustomResourceDefinition{
2526
manifests.ReferenceGrantCRD(),
2627
}
2728

29+
// managedCRDMap is a map of CRDs that this controller manages.
30+
var managedCRDMap = map[string]*apiextensionsv1.CustomResourceDefinition{
31+
manifests.GatewayClassCRD().Name: manifests.GatewayClassCRD(),
32+
manifests.GatewayCRD().Name: manifests.GatewayCRD(),
33+
manifests.GRPCRouteCRD().Name: manifests.GRPCRouteCRD(),
34+
manifests.HTTPRouteCRD().Name: manifests.HTTPRouteCRD(),
35+
manifests.ReferenceGrantCRD().Name: manifests.ReferenceGrantCRD(),
36+
}
37+
2838
// ensureCRD attempts to ensure that the specified CRD exists and returns a
2939
// Boolean indicating whether it exists, the CRD if it does exist, and an error
3040
// value.
@@ -69,6 +79,24 @@ func (r *reconciler) ensureGatewayAPICRDs(ctx context.Context) error {
6979
return utilerrors.NewAggregate(errs)
7080
}
7181

82+
// listUnmanagedGatewayAPICRDs returns a list of unmanaged Gateway API CRDs
83+
// which exist in the cluster. A Gateway API CRD has "gateway.networking.k8s.io"
84+
// or "gateway.networking.x-k8s.io" in its "spec.group" field.
85+
func (r *reconciler) listUnmanagedGatewayAPICRDs(ctx context.Context) ([]string, error) {
86+
gatewayAPICRDs := &apiextensionsv1.CustomResourceDefinitionList{}
87+
if err := r.cache.List(ctx, gatewayAPICRDs, client.MatchingFields{crdAPIGroupIndexFieldName: gatewayCRDAPIGroupIndexFieldValue}); err != nil {
88+
return nil, fmt.Errorf("failed to list gateway API CRDs: %w", err)
89+
}
90+
91+
var unmanagedCRDNames []string
92+
for _, crd := range gatewayAPICRDs.Items {
93+
if _, found := managedCRDMap[crd.Name]; !found {
94+
unmanagedCRDNames = append(unmanagedCRDNames, crd.Name)
95+
}
96+
}
97+
return unmanagedCRDNames, nil
98+
}
99+
72100
// currentCRD returns a Boolean indicating whether an CRD
73101
// exists for the IngressController with the given name, as well as the
74102
// CRD if it does exist and an error value.

0 commit comments

Comments
 (0)