Skip to content

Commit 844683f

Browse files
author
Ali Syed
committed
NE-1951: Pre-upgrade Admin Gate for Gateway API CRD Management Succession
This PR introduces a pre-upgrade admin gate for Gateway API CRD management succession. It implements an operator which, upon the detection of any Gateway API CRDs will apply an admin gate to inform the cluster admin. There is also an e2e implemented and a conditional check for the operator to skip if the featuregate is enabled. This ensures to block upgrades until a cluster admin explicitly provides consent for the platform to start taking over management of the Gateway API CRDs
1 parent a730e8a commit 844683f

File tree

5 files changed

+269
-1
lines changed

5 files changed

+269
-1
lines changed
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
package gatewayapi_upgradeable
2+
3+
import (
4+
"context"
5+
"fmt"
6+
logf "github.com/openshift/cluster-ingress-operator/pkg/log"
7+
operatorcontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller"
8+
9+
corev1 "k8s.io/api/core/v1"
10+
11+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
12+
13+
"sigs.k8s.io/controller-runtime/pkg/cache"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/controller"
16+
"sigs.k8s.io/controller-runtime/pkg/handler"
17+
"sigs.k8s.io/controller-runtime/pkg/manager"
18+
"sigs.k8s.io/controller-runtime/pkg/predicate"
19+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
20+
"sigs.k8s.io/controller-runtime/pkg/source"
21+
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"
22+
)
23+
24+
const (
25+
controllerName = "gatewayapi_upgradeable_controller"
26+
gatewayAPIAdminKey = "ack-gateway-api-management"
27+
gatewayAPIAdminMsg = "Gateway API CRDs have been detected. OCP fully manages the life-cycle of Gateway API CRDs. External management is unsupported and will be prevented. The cluster administrator is responsible for the safety of existing Gateway API implementations and must acknowledge their responsibilities via the admin gate to proceed with upgrades. See https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/release_notes/ocp-4-19-release-notes#ocp-4-19-networking-gateway-api-crd-lifecycle_release-notes for details. Failure to read and understand the documentation for this and the implications can result in outages and data loss."
28+
)
29+
30+
var (
31+
log = logf.Logger.WithName(controllerName)
32+
)
33+
34+
// The New function initializes the controller and sets up the watch for the ConfigMap.
35+
func New(mgr manager.Manager) (controller.Controller, error) {
36+
c, err := controller.New(controllerName, mgr, controller.Options{
37+
Reconciler: &reconciler{
38+
client: mgr.GetClient(),
39+
cache: mgr.GetCache(),
40+
},
41+
})
42+
if err != nil {
43+
return nil, err
44+
}
45+
46+
// Mapping the events to the admin gate ConfigMap.
47+
toAdminGatesConfigMap := func(_ context.Context, _ client.Object) []reconcile.Request {
48+
return []reconcile.Request{{
49+
NamespacedName: operatorcontroller.AdminGatesConfigMapName(),
50+
}}
51+
}
52+
53+
// Defining the CRD predicate.
54+
crdPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
55+
group := o.(*apiextensionsv1.CustomResourceDefinition).Spec.Group
56+
return group == gatewayapiv1.GroupName || group == "gateway.networking.x-k8s.io"
57+
})
58+
59+
// Setting up a watch for CRD events.
60+
if err := c.Watch(source.Kind[client.Object](mgr.GetCache(), &apiextensionsv1.CustomResourceDefinition{}, handler.EnqueueRequestsFromMapFunc(toAdminGatesConfigMap), crdPredicate)); err != nil {
61+
return nil, err
62+
}
63+
64+
// A predicate filter to watch for specific changes in the ConfigMap.
65+
// Verify that the ConfigMap's name and namespace match the expected values.
66+
adminGatePredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
67+
return o.GetNamespace() == operatorcontroller.AdminGatesConfigMapName().Namespace &&
68+
o.GetName() == operatorcontroller.AdminGatesConfigMapName().Name
69+
})
70+
71+
if err := c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, &handler.EnqueueRequestForObject{}, adminGatePredicate)); err != nil {
72+
return nil, err
73+
}
74+
75+
return c, nil
76+
}
77+
78+
// Reconciler struct holds the client and cache attributes.
79+
type reconciler struct {
80+
client client.Client
81+
cache cache.Cache
82+
}
83+
84+
// Reconcile function implements the logic to check conditions and manage the admin gate.
85+
func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
86+
log.Info("reconciling", "request", request)
87+
88+
adminGateConditionExists, err := r.adminGateConditionExists(ctx)
89+
if err != nil {
90+
return reconcile.Result{}, fmt.Errorf("failed to determine if admin gate condition exists: %w", err)
91+
}
92+
93+
if adminGateConditionExists {
94+
if err := r.addAdminGate(ctx); err != nil {
95+
return reconcile.Result{}, fmt.Errorf("failed to add admin gate: %w", err)
96+
}
97+
} else {
98+
if err := r.removeAdminGate(ctx); err != nil {
99+
return reconcile.Result{}, fmt.Errorf("failed to remove admin gate: %w", err)
100+
}
101+
}
102+
103+
return reconcile.Result{}, nil
104+
}
105+
106+
// adminGateConditionExists checks if the admin gate condition exists based on both ConfigMap and CRDs.
107+
func (r *reconciler) adminGateConditionExists(ctx context.Context) (bool, error) {
108+
crds := &apiextensionsv1.CustomResourceDefinitionList{}
109+
if err := r.cache.List(ctx, crds); err != nil {
110+
return false, fmt.Errorf("failed to list CRDs: %w", err)
111+
}
112+
113+
for _, crd := range crds.Items {
114+
if crd.Spec.Group == gatewayapiv1.GroupName || crd.Spec.Group == "gateway.networking.x-k8s.io" {
115+
return true, nil
116+
}
117+
}
118+
119+
return false, nil
120+
}
121+
122+
// The addAdminGate function is responsible for adding the admin gate to the ConfigMap.
123+
func (r *reconciler) addAdminGate(ctx context.Context) error {
124+
adminGatesConfigMap := &corev1.ConfigMap{}
125+
if err := r.cache.Get(ctx, operatorcontroller.AdminGatesConfigMapName(), adminGatesConfigMap); err != nil {
126+
return fmt.Errorf("failed to get configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
127+
}
128+
129+
if adminGatesConfigMap.Data == nil {
130+
adminGatesConfigMap.Data = map[string]string{}
131+
}
132+
133+
// The function checks if the admin key exists and if it is set to the expected message.
134+
if val, ok := adminGatesConfigMap.Data[gatewayAPIAdminKey]; ok && val == gatewayAPIAdminMsg {
135+
return nil
136+
}
137+
adminGatesConfigMap.Data[gatewayAPIAdminKey] = gatewayAPIAdminMsg
138+
139+
log.Info("Adding admin gate for Gateway API management")
140+
if err := r.client.Update(ctx, adminGatesConfigMap); err != nil {
141+
return fmt.Errorf("failed to update configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
142+
}
143+
return nil
144+
}
145+
146+
// The removeAdminGate function is responsible for removing the admin gate from the ConfigMap.
147+
func (r *reconciler) removeAdminGate(ctx context.Context) error {
148+
adminGatesConfigMap := &corev1.ConfigMap{}
149+
if err := r.cache.Get(ctx, operatorcontroller.AdminGatesConfigMapName(), adminGatesConfigMap); err != nil {
150+
return fmt.Errorf("failed to get configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
151+
}
152+
153+
if _, ok := adminGatesConfigMap.Data[gatewayAPIAdminKey]; !ok {
154+
return nil
155+
}
156+
157+
log.Info("Removing admin gate for Gateway API management")
158+
delete(adminGatesConfigMap.Data, gatewayAPIAdminKey)
159+
if err := r.client.Update(ctx, adminGatesConfigMap); err != nil {
160+
return fmt.Errorf("failed to update configmap %s: %w", operatorcontroller.AdminGatesConfigMapName(), err)
161+
}
162+
return nil
163+
}

pkg/operator/controller/names.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,13 @@ const (
4949
RemoteWorkerLabel = "node.openshift.io/remote-worker"
5050
)
5151

52+
func AdminGatesConfigMapName() types.NamespacedName {
53+
return types.NamespacedName{
54+
Name: "admin-gates",
55+
Namespace: GlobalMachineSpecifiedConfigNamespace,
56+
}
57+
}
58+
5259
// IngressClusterOperatorName returns the namespaced name of the ClusterOperator
5360
// resource for the operator.
5461
func IngressClusterOperatorName() types.NamespacedName {

pkg/operator/operator.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ import (
3535
dnscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/dns"
3636
gatewayservicednscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gateway-service-dns"
3737
gatewayapicontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gatewayapi"
38+
gatewayapi_upgradeable "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gatewayapi-upgradeable"
3839
gatewayclasscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/gatewayclass"
3940
ingress "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress"
4041
ingresscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress"
4142
ingressclasscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingressclass"
4243
statuscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/status"
4344
"github.com/openshift/library-go/pkg/operator/events"
44-
4545
"k8s.io/apimachinery/pkg/api/errors"
4646
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
4747
"k8s.io/apimachinery/pkg/types"
@@ -319,6 +319,13 @@ func New(config operatorconfig.Config, kubeConfig *rest.Config) (*Operator, erro
319319
return nil, fmt.Errorf("failed to create gatewayapi controller: %w", err)
320320
}
321321

322+
// Add conditional setup for gateway_upgradeable controller only if FeatureGate is not enabled
323+
if !gatewayAPIEnabled {
324+
if _, err := gatewayapi_upgradeable.New(mgr); err != nil {
325+
return nil, fmt.Errorf("failed to create gatewayapi upgradeable controller: %w", err)
326+
}
327+
}
328+
322329
return &Operator{
323330
manager: mgr,
324331
// TODO: These are only needed for the default ingress controller stuff, which

test/e2e/all_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ func TestAll(t *testing.T) {
9090
t.Run("TestUnmanagedAWSEIPAllocations", TestUnmanagedAWSEIPAllocations)
9191
t.Run("Test_IdleConnectionTerminationPolicyImmediate", Test_IdleConnectionTerminationPolicyImmediate)
9292
t.Run("Test_IdleConnectionTerminationPolicyDeferred", Test_IdleConnectionTerminationPolicyDeferred)
93+
t.Run("TestGatewayAPIUpgradeable", TestGatewayAPIUpgradeable)
9394
})
9495

9596
t.Run("serial", func(t *testing.T) {
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
package e2e
5+
6+
import (
7+
"context"
8+
"testing"
9+
"time"
10+
11+
corev1 "k8s.io/api/core/v1"
12+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"
15+
16+
"github.com/openshift/api/features"
17+
18+
"k8s.io/apimachinery/pkg/util/wait"
19+
"sigs.k8s.io/controller-runtime/pkg/client"
20+
)
21+
22+
func TestGatewayAPIUpgradeable(t *testing.T) {
23+
t.Parallel()
24+
if gatewayAPIEnabled, err := isFeatureGateEnabled(features.FeatureGateGatewayAPI); err != nil {
25+
t.Fatalf("error checking feature gate enabled status: %v", err)
26+
} else if gatewayAPIEnabled {
27+
t.Skip("Gateway API is enabled, skipping TestGatewayAPIUpgradeable")
28+
}
29+
createCRDs(t)
30+
testAdminGate(t, true)
31+
deleteExistingCRD(t, "gatewayclasses.gateway.networking.k8s.io")
32+
testAdminGate(t, false)
33+
34+
}
35+
36+
func createCRDs(t *testing.T) {
37+
t.Helper()
38+
crd := &apiextensionsv1.CustomResourceDefinition{
39+
ObjectMeta: metav1.ObjectMeta{
40+
Name: "gatewayclasses.gateway.networking.k8s.io",
41+
Annotations: map[string]string{
42+
"api-approved.kubernetes.io": "https://github.com/kubernetes-sigs/gateway-api/pull/2466",
43+
},
44+
},
45+
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
46+
Group: gatewayapiv1.GroupName,
47+
Names: apiextensionsv1.CustomResourceDefinitionNames{
48+
Singular: "gatewayclass",
49+
Plural: "gatewayclasses",
50+
Kind: "GatewayClass",
51+
},
52+
Scope: apiextensionsv1.ClusterScoped,
53+
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
54+
{
55+
Name: "v1",
56+
Storage: true,
57+
Served: true,
58+
Schema: &apiextensionsv1.CustomResourceValidation{
59+
OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
60+
Type: "object",
61+
},
62+
},
63+
},
64+
},
65+
},
66+
}
67+
68+
if err := kclient.Create(context.TODO(), crd); err != nil {
69+
t.Fatalf("Failed to create CRD %s: %v", crd.Name, err)
70+
}
71+
}
72+
73+
func testAdminGate(t *testing.T, shouldExist bool) {
74+
t.Helper()
75+
adminGatesConfigMap := &corev1.ConfigMap{}
76+
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 30*time.Second, false, func(ctx context.Context) (bool, error) {
77+
if err := kclient.Get(ctx, client.ObjectKey{Namespace: "openshift-config-managed", Name: "admin-gates"}, adminGatesConfigMap); err != nil {
78+
t.Logf("Failed to get configmap admin-gates: %v, retrying...", err)
79+
return false, nil
80+
}
81+
return true, nil
82+
}); err != nil {
83+
t.Fatalf("Timed out trying to get admin-gates configmap: %v", err)
84+
}
85+
86+
_, adminGateKeyExists := adminGatesConfigMap.Data["ack-gateway-api-management"]
87+
if adminGateKeyExists != shouldExist {
88+
t.Fatalf("Expected admin gate key existence to be %v, but got %v", shouldExist, adminGateKeyExists)
89+
}
90+
}

0 commit comments

Comments
 (0)