Skip to content

Commit d967a73

Browse files
Merge pull request #1196 from Thealisyed/gwapi-admingate-4.18
NE-1951: Pre-upgrade Admin Gate for Gateway API CRD Management Succession
2 parents b0222ed + c22c21d commit d967a73

File tree

4 files changed

+220
-7
lines changed

4 files changed

+220
-7
lines changed
Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
package gatewayapi_upgradeable
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
logf "github.com/openshift/cluster-ingress-operator/pkg/log"
8+
operatorcontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller"
9+
10+
corev1 "k8s.io/api/core/v1"
11+
12+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
13+
14+
"sigs.k8s.io/controller-runtime/pkg/cache"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
"sigs.k8s.io/controller-runtime/pkg/controller"
17+
"sigs.k8s.io/controller-runtime/pkg/handler"
18+
"sigs.k8s.io/controller-runtime/pkg/manager"
19+
"sigs.k8s.io/controller-runtime/pkg/predicate"
20+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
21+
"sigs.k8s.io/controller-runtime/pkg/source"
22+
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"
23+
)
24+
25+
const (
26+
controllerName = "gatewayapi_upgradeable_controller"
27+
gatewayAPIAdminKey = "ack-4.18-gateway-api-management-in-4.19"
28+
gatewayAPIAdminMsg = "Gateway API CRDs have been detected. OCP fully manages the life-cycle of Gateway API CRDs. External management is unsupported and will be prevented. The cluster administrator is responsible for the safety of existing Gateway API implementations and must acknowledge their responsibilities via the admin gate to proceed with upgrades. See https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/release_notes/ocp-4-19-release-notes#ocp-4-19-networking-gateway-api-crd-lifecycle_release-notes for details. Failure to read and understand the documentation for this and the implications can result in outages and data loss."
29+
)
30+
31+
var (
32+
log = logf.Logger.WithName(controllerName)
33+
)
34+
35+
// The New function initializes the controller and sets up the watch for the ConfigMap.
36+
func New(mgr manager.Manager) (controller.Controller, error) {
37+
c, err := controller.New(controllerName, mgr, controller.Options{
38+
Reconciler: &reconciler{
39+
client: mgr.GetClient(),
40+
cache: mgr.GetCache(),
41+
},
42+
})
43+
if err != nil {
44+
return nil, err
45+
}
46+
47+
// Mapping the events to the admin gate ConfigMap.
48+
toAdminGatesConfigMap := func(_ context.Context, _ client.Object) []reconcile.Request {
49+
return []reconcile.Request{{
50+
NamespacedName: operatorcontroller.AdminGatesConfigMapName(),
51+
}}
52+
}
53+
54+
// Defining the CRD predicate.
55+
crdPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
56+
group := o.(*apiextensionsv1.CustomResourceDefinition).Spec.Group
57+
return group == gatewayapiv1.GroupName || group == "gateway.networking.x-k8s.io"
58+
})
59+
60+
// Setting up a watch for CRD events.
61+
if err := c.Watch(source.Kind[client.Object](mgr.GetCache(), &apiextensionsv1.CustomResourceDefinition{}, handler.EnqueueRequestsFromMapFunc(toAdminGatesConfigMap), crdPredicate)); err != nil {
62+
return nil, err
63+
}
64+
65+
// A predicate filter to watch for specific changes in the ConfigMap.
66+
// Verify that the ConfigMap's name and namespace match the expected values.
67+
adminGatePredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
68+
return o.GetNamespace() == operatorcontroller.AdminGatesConfigMapName().Namespace &&
69+
o.GetName() == operatorcontroller.AdminGatesConfigMapName().Name
70+
})
71+
72+
if err := c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, &handler.EnqueueRequestForObject{}, adminGatePredicate)); err != nil {
73+
return nil, err
74+
}
75+
76+
return c, nil
77+
}
78+
79+
// reconciler struct holds the client and cache attributes.
80+
type reconciler struct {
81+
client client.Client
82+
cache cache.Cache
83+
}
84+
85+
// Reconcile function implements the logic to check conditions and manage the admin gate.
86+
func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
87+
log.Info("reconciling", "request", request)
88+
89+
adminGateConditionExists, err := r.adminGateConditionExists(ctx)
90+
if err != nil {
91+
return reconcile.Result{}, fmt.Errorf("failed to determine if admin gate condition exists: %w", err)
92+
}
93+
94+
if adminGateConditionExists {
95+
if err := r.addAdminGate(ctx); err != nil {
96+
return reconcile.Result{}, fmt.Errorf("failed to add admin gate: %w", err)
97+
}
98+
} else {
99+
if err := r.removeAdminGate(ctx); err != nil {
100+
return reconcile.Result{}, fmt.Errorf("failed to remove admin gate: %w", err)
101+
}
102+
}
103+
104+
return reconcile.Result{}, nil
105+
}
106+
107+
// adminGateConditionExists checks if the admin gate condition exists based on both ConfigMap and CRDs.
108+
func (r *reconciler) adminGateConditionExists(ctx context.Context) (bool, error) {
109+
crds := &apiextensionsv1.CustomResourceDefinitionList{}
110+
if err := r.cache.List(ctx, crds); err != nil {
111+
return false, fmt.Errorf("failed to list CRDs: %w", err)
112+
}
113+
114+
for _, crd := range crds.Items {
115+
if crd.Spec.Group == gatewayapiv1.GroupName || crd.Spec.Group == "gateway.networking.x-k8s.io" {
116+
return true, nil
117+
}
118+
}
119+
120+
return false, nil
121+
}
122+
123+
// The addAdminGate function is responsible for adding the admin gate to the ConfigMap.
124+
func (r *reconciler) addAdminGate(ctx context.Context) error {
125+
adminGatesConfigMap := &corev1.ConfigMap{}
126+
if err := r.cache.Get(ctx, operatorcontroller.AdminGatesConfigMapName(), adminGatesConfigMap); err != nil {
127+
return fmt.Errorf("failed to get configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
128+
}
129+
130+
if adminGatesConfigMap.Data == nil {
131+
adminGatesConfigMap.Data = map[string]string{}
132+
}
133+
134+
// The function checks if the admin key exists and if it is set to the expected message.
135+
if val, ok := adminGatesConfigMap.Data[gatewayAPIAdminKey]; ok && val == gatewayAPIAdminMsg {
136+
return nil
137+
}
138+
adminGatesConfigMap.Data[gatewayAPIAdminKey] = gatewayAPIAdminMsg
139+
140+
log.Info("Adding admin gate for Gateway API management")
141+
if err := r.client.Update(ctx, adminGatesConfigMap); err != nil {
142+
return fmt.Errorf("failed to update configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
143+
}
144+
return nil
145+
}
146+
147+
// The removeAdminGate function is responsible for removing the admin gate from the ConfigMap.
148+
func (r *reconciler) removeAdminGate(ctx context.Context) error {
149+
adminGatesConfigMap := &corev1.ConfigMap{}
150+
if err := r.cache.Get(ctx, operatorcontroller.AdminGatesConfigMapName(), adminGatesConfigMap); err != nil {
151+
return fmt.Errorf("failed to get configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
152+
}
153+
154+
if _, ok := adminGatesConfigMap.Data[gatewayAPIAdminKey]; !ok {
155+
return nil
156+
}
157+
158+
log.Info("Removing admin gate for Gateway API management")
159+
delete(adminGatesConfigMap.Data, gatewayAPIAdminKey)
160+
if err := r.client.Update(ctx, adminGatesConfigMap); err != nil {
161+
return fmt.Errorf("failed to update configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
162+
}
163+
return nil
164+
}

pkg/operator/controller/names.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,13 @@ const (
4949
RemoteWorkerLabel = "node.openshift.io/remote-worker"
5050
)
5151

52+
func AdminGatesConfigMapName() types.NamespacedName {
53+
return types.NamespacedName{
54+
Name: "admin-gates",
55+
Namespace: GlobalMachineSpecifiedConfigNamespace,
56+
}
57+
}
58+
5259
// IngressClusterOperatorName returns the namespaced name of the ClusterOperator
5360
// resource for the operator.
5461
func IngressClusterOperatorName() types.NamespacedName {

pkg/operator/operator.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ import (
3535
dnscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/dns"
3636
gatewayservicednscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gateway-service-dns"
3737
gatewayapicontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gatewayapi"
38+
gatewayapi_upgradeable "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gatewayapi-upgradeable"
3839
gatewayclasscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gatewayclass"
3940
ingress "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress"
4041
ingresscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress"
4142
ingressclasscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingressclass"
4243
statuscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/status"
4344
"github.com/openshift/library-go/pkg/operator/events"
44-
4545
"k8s.io/apimachinery/pkg/api/errors"
4646
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
4747
"k8s.io/apimachinery/pkg/types"
@@ -320,6 +320,13 @@ func New(config operatorconfig.Config, kubeConfig *rest.Config) (*Operator, erro
320320
return nil, fmt.Errorf("failed to create gatewayapi controller: %w", err)
321321
}
322322

323+
// Add conditional setup for gateway_upgradeable controller only if FeatureGate is not enabled
324+
if !gatewayAPIEnabled {
325+
if _, err := gatewayapi_upgradeable.New(mgr); err != nil {
326+
return nil, fmt.Errorf("failed to create gatewayapi upgradeable controller: %w", err)
327+
}
328+
}
329+
323330
return &Operator{
324331
manager: mgr,
325332
// TODO: These are only needed for the default ingress controller stuff, which

test/e2e/gatewayapi_upgradeable_test.go

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,18 @@ package e2e
66
import (
77
"context"
88
"testing"
9-
10-
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
9+
"time"
1110

1211
configv1 "github.com/openshift/api/config/v1"
1312
"github.com/openshift/api/features"
1413
"github.com/openshift/cluster-ingress-operator/pkg/manifests"
1514
test_crds "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/test/crds"
1615

16+
corev1 "k8s.io/api/core/v1"
17+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1718
"k8s.io/apimachinery/pkg/api/errors"
19+
"k8s.io/apimachinery/pkg/util/wait"
20+
"sigs.k8s.io/controller-runtime/pkg/client"
1821
)
1922

2023
var expectedCRDs = []*apiextensionsv1.CustomResourceDefinition{
@@ -30,21 +33,35 @@ var incompatibleCRDs = []*apiextensionsv1.CustomResourceDefinition{
3033
test_crds.TCPRouteCRD_experimental_v1(),
3134
}
3235

36+
// TestGatewayAPIUpgradeable verifies the operator's upgradeable condition and admin gate behavior
37+
// when the Gateway API feature gate is disabled.
3338
func TestGatewayAPIUpgradeable(t *testing.T) {
39+
t.Parallel()
3440
if gatewayAPIEnabled, err := isFeatureGateEnabled(features.FeatureGateGatewayAPI); err != nil {
3541
t.Fatalf("error checking feature gate enabled status: %v", err)
3642
} else if gatewayAPIEnabled {
3743
t.Skip("Gateway API is enabled, skipping TestGatewayAPIUpgradeable")
3844
}
3945

40-
t.Parallel()
41-
42-
defer deleteExistingCRDs(t, append(expectedCRDs, incompatibleCRDs...))
46+
t.Cleanup(func() {
47+
for _, crd := range append(expectedCRDs, incompatibleCRDs...) {
48+
if err := kclient.Delete(context.TODO(), crd); err != nil {
49+
if errors.IsNotFound(err) {
50+
continue
51+
}
52+
t.Errorf("failed to delete crd %q: %v", crd.Name, err)
53+
}
54+
}
55+
})
4356

4457
createCRDs(t, expectedCRDs)
58+
testAdminGate(t, true)
4559
testOperatorUpgradeableCondition(t, true)
4660
createCRDs(t, incompatibleCRDs)
4761
testOperatorUpgradeableCondition(t, false)
62+
deleteExistingCRDs(t, append(expectedCRDs, incompatibleCRDs...))
63+
testAdminGate(t, false)
64+
testOperatorUpgradeableCondition(t, true)
4865
}
4966

5067
func testOperatorUpgradeableCondition(t *testing.T, expectUpgradeable bool) {
@@ -69,7 +86,6 @@ func createCRDs(t *testing.T, crds []*apiextensionsv1.CustomResourceDefinition)
6986
if !errors.IsAlreadyExists(err) {
7087
t.Fatalf("Failed to create CRD %s: %v", crd.Name, err)
7188
}
72-
continue
7389
}
7490
}
7591
}
@@ -80,3 +96,22 @@ func deleteExistingCRDs(t *testing.T, crds []*apiextensionsv1.CustomResourceDefi
8096
deleteExistingCRD(t, crd.Name)
8197
}
8298
}
99+
100+
func testAdminGate(t *testing.T, shouldExist bool) {
101+
t.Helper()
102+
adminGatesConfigMap := &corev1.ConfigMap{}
103+
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 30*time.Second, false, func(ctx context.Context) (bool, error) {
104+
if err := kclient.Get(ctx, client.ObjectKey{Namespace: "openshift-config-managed", Name: "admin-gates"}, adminGatesConfigMap); err != nil {
105+
t.Logf("Failed to get configmap admin-gates: %v, retrying...", err)
106+
return false, nil
107+
}
108+
return true, nil
109+
}); err != nil {
110+
t.Fatalf("Timed out trying to get admin-gates configmap: %v", err)
111+
}
112+
113+
_, adminGateKeyExists := adminGatesConfigMap.Data["ack-4.18-gateway-api-management-in-4.19"]
114+
if adminGateKeyExists != shouldExist {
115+
t.Fatalf("Expected admin gate key existence to be %v, but got %v", shouldExist, adminGateKeyExists)
116+
}
117+
}

0 commit comments

Comments
 (0)