Skip to content

Commit 63f540a

Browse files
committed
NE-2097: e2e - Add cleanup helpers for GatewayAPI resources
This commit introduces helper functions to clean up Gateway, GatewayClass, and OLM operator after e2e tests. These functions are integrated into the GatewayAPI and GatewayAPI upgrade tests to prevent conflicts and ensure a clean environment for subsequent runs.
1 parent 0d03c96 commit 63f540a

File tree

12 files changed

+1162
-52
lines changed

12 files changed

+1162
-52
lines changed

test/e2e/gateway_api_test.go

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,16 @@ func TestGatewayAPI(t *testing.T) {
8383

8484
// Defer the cleanup of the test gateway.
8585
t.Cleanup(func() {
86-
testGateway := gatewayapiv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: testGatewayName, Namespace: operatorcontroller.DefaultOperandNamespace}}
87-
if err := kclient.Delete(context.TODO(), &testGateway); err != nil {
88-
if errors.IsNotFound(err) {
89-
return
90-
}
91-
t.Errorf("failed to delete gateway %q: %v", testGateway.Name, err)
86+
t.Logf("Cleaning Gateway, GatewayClass and OSSM operator...")
87+
if err := cleanupGateway(t, "openshift-ingress", "test-gateway", "openshift-default"); err != nil {
88+
t.Errorf("Failed to cleanup gateway: %v", err)
89+
}
90+
if err := cleanupGatewayClass(t, "openshift-default"); err != nil {
91+
t.Errorf("Failed to cleanup gatewayclass: %v", err)
92+
}
93+
if err := cleanupOLMOperator(t, "openshift-operators", "servicemeshoperator3.openshift-operators"); err != nil {
94+
t.Errorf("Failed to cleanup OSSM operator: %v", err)
9295
}
93-
// TODO: Uninstall OSSM after test is completed.
9496
})
9597

9698
t.Run("testGatewayAPIResources", testGatewayAPIResources)
@@ -547,10 +549,10 @@ func testGatewayAPIDNS(t *testing.T) {
547549
// Create gateways
548550
for _, gateway := range tc.createGateways {
549551
createdGateway, err := createGatewayWithListeners(gatewayClass, gateway.gatewayName, gateway.namespace, gateway.listeners)
550-
gateways = append(gateways, createdGateway)
551552
if err != nil {
552553
t.Fatalf("failed to create gateway %s: %v", gateway.gatewayName, err)
553554
}
555+
gateways = append(gateways, createdGateway)
554556
}
555557

556558
t.Cleanup(func() {

test/e2e/gateway_api_upgrade_test.go

Lines changed: 13 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010

1111
"k8s.io/apimachinery/pkg/types"
1212
"k8s.io/apimachinery/pkg/util/wait"
13-
"k8s.io/apiserver/pkg/storage/names"
1413
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"
1514

1615
configv1 "github.com/openshift/api/config/v1"
@@ -35,6 +34,16 @@ func TestOSSMOperatorUpgradeViaIntermediateVersions(t *testing.T) {
3534
t.Skip("Gateway API featuregates are not enabled, skipping TestOSSMOperatorUpgradeViaIntermediateVersions")
3635
}
3736

37+
t.Cleanup(func() {
38+
t.Logf("Cleaning GatewayClass and OSSM operator...")
39+
if err := cleanupGatewayClass(t, "openshift-default"); err != nil {
40+
t.Errorf("Failed to cleanup gatewayclass: %v", err)
41+
}
42+
if err := cleanupOLMOperator(t, "openshift-operators", "servicemeshoperator3.openshift-operators"); err != nil {
43+
t.Errorf("Failed to cleanup OSSM operator: %v", err)
44+
}
45+
})
46+
3847
var (
3948
initialOSSMVersion = "servicemeshoperator3.v3.0.0"
4049
initialIstioVersion = "v1.24.3"
@@ -61,7 +70,7 @@ func TestOSSMOperatorUpgradeViaIntermediateVersions(t *testing.T) {
6170
t.Fatalf("Failed to find expected CatalogSource %s: %v", expectedCatalogSourceName, err)
6271
}
6372
t.Log("Checking for the OSSM operator deployment...")
64-
if err := assertOSSMOperatorWithConfig(t, initialOSSMVersion, 2*time.Second, 60*time.Second); err != nil {
73+
if err := assertOSSMOperatorWithConfig(t, initialOSSMVersion, 2*time.Second, 2*time.Minute); err != nil {
6574
t.Fatalf("Failed to find expected Istio operator: %v", err)
6675
}
6776
t.Log("Checking for the Istio CR...")
@@ -99,7 +108,7 @@ func TestOSSMOperatorUpgradeViaIntermediateVersions(t *testing.T) {
99108
{
100109
Type: configv1.OperatorProgressing,
101110
Status: configv1.ConditionTrue,
102-
Reason: "OSSMOperatorUpgrading",
111+
Reason: "GatewayAPIOperatorUpgrading",
103112
},
104113
{
105114
Type: configv1.OperatorDegraded,
@@ -119,7 +128,7 @@ func TestOSSMOperatorUpgradeViaIntermediateVersions(t *testing.T) {
119128
{
120129
Type: configv1.OperatorProgressing,
121130
Status: configv1.ConditionFalse,
122-
Reason: "AsExpectedAndOSSMOperatorUpToDate",
131+
Reason: "AsExpectedAndGatewayAPIOperatorUpToDate",
123132
},
124133
{
125134
Type: configv1.OperatorDegraded,
@@ -139,43 +148,3 @@ func TestOSSMOperatorUpgradeViaIntermediateVersions(t *testing.T) {
139148
t.Fatalf("Failed to find expected Istio: %v", err)
140149
}
141150
}
142-
143-
// testConnectivity tests the connectivity to an HTTPRoute.
144-
func testConnectivity(t *testing.T, gatewayClass *gatewayapiv1.GatewayClass) {
145-
t.Helper()
146-
147-
// Use the dnsConfig base domain set up in TestMain.
148-
domain := "gws." + dnsConfig.Spec.BaseDomain
149-
150-
t.Log("Creating gateway...")
151-
testGateway, err := createGateway(gatewayClass, "test-upgrade-gateway", "openshift-ingress", domain)
152-
if err != nil {
153-
t.Fatalf("Gateway could not be created: %v", err)
154-
}
155-
// The gateway is cleaned up in TestGatewayAPI.
156-
157-
// Create a test namespace that cleans itself up and sets up its own service account and role binding.
158-
ns := createNamespace(t, names.SimpleNameGenerator.GenerateName("test-e2e-gwapi-upgrade-"))
159-
routeName := names.SimpleNameGenerator.GenerateName("test-hostname-") + "." + domain
160-
161-
t.Log("Creating httproute...")
162-
if _, err = createHttpRoute(ns.Name, "test-httproute", "openshift-ingress", routeName, "test-upgrade-gateway-openshift-default", testGateway); err != nil {
163-
t.Fatalf("HTTPRoute could not be created: %v", err)
164-
}
165-
// The http route is cleaned up when the namespace is deleted.
166-
167-
t.Log("Making sure the gateway is accepted...")
168-
if _, err := assertGatewaySuccessful(t, testGateway.Namespace, testGateway.Name); err != nil {
169-
t.Fatalf("Failed to find successful gateway: %v", err)
170-
}
171-
172-
t.Log("Making sure the httproute is accepted...")
173-
if _, err := assertHttpRouteSuccessful(t, ns.Name, "test-httproute", testGateway); err != nil {
174-
t.Fatalf("Failed to find successful httproute: %v", err)
175-
}
176-
177-
t.Log("Validating the connectivity to the backend application via the httproute...")
178-
if err := assertHttpRouteConnection(t, routeName, testGateway); err != nil {
179-
t.Fatalf("Failed to find successful httproute: %v", err)
180-
}
181-
}

test/e2e/operator_test.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
operatorv1 "github.com/openshift/api/operator/v1"
3232
iov1 "github.com/openshift/api/operatoringress/v1"
3333
routev1 "github.com/openshift/api/route/v1"
34+
olmv1 "github.com/operator-framework/api/pkg/operators/v1"
3435

3536
configclientset "github.com/openshift/client-go/config/clientset/versioned"
3637
"github.com/openshift/cluster-ingress-operator/pkg/manifests"
@@ -181,6 +182,12 @@ func TestMain(m *testing.M) {
181182
}
182183
kclient = kubeClient
183184

185+
// GatewayAPI tests need to clean up the OSSM operator.
186+
if olmv1.AddToScheme(operatorclient.GetScheme()); err != nil {
187+
fmt.Printf("failed to install olmv1 scheme: %s\n", err)
188+
os.Exit(1)
189+
}
190+
184191
configClient, err = configclientset.NewForConfig(kubeConfig)
185192
if err != nil {
186193
fmt.Printf("failed to create config client: %s\n", err)

test/e2e/util_gatewayapi_test.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
v1 "github.com/openshift/api/operatoringress/v1"
2121
operatorcontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller"
2222
util "github.com/openshift/cluster-ingress-operator/pkg/util"
23+
olmv1 "github.com/operator-framework/api/pkg/operators/v1"
2324
operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
2425

2526
"github.com/google/go-cmp/cmp"
@@ -30,6 +31,7 @@ import (
3031
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
3132
kerrors "k8s.io/apimachinery/pkg/api/errors"
3233
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3335
"k8s.io/apimachinery/pkg/types"
3436
"k8s.io/apimachinery/pkg/util/wait"
3537

@@ -180,6 +182,106 @@ func deleteExistingVAP(t *testing.T, vapName string) error {
180182
return nil
181183
}
182184

185+
// cleanupGateway deletes the given gateway and waits for the corresponding Istio proxy deployment to disappear.
186+
func cleanupGateway(t *testing.T, namespace, name, gcname string) error {
187+
t.Helper()
188+
189+
gw := &gatewayapiv1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}
190+
if err := kclient.Delete(context.Background(), gw); err != nil && !kerrors.IsNotFound(err) {
191+
return fmt.Errorf(`failed to delete gateway "%s/%s": %w`, namespace, name, err)
192+
}
193+
t.Logf(`Deleted gateway "%s/%s"`, namespace, name)
194+
195+
depl := &appsv1.Deployment{}
196+
istioName := types.NamespacedName{Namespace: operatorcontroller.DefaultOperandNamespace, Name: name + "-" + gcname}
197+
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 1*time.Minute, false, func(context context.Context) (bool, error) {
198+
if err := kclient.Get(context, istioName, depl); err != nil {
199+
if !kerrors.IsNotFound(err) {
200+
t.Logf("Failed to get deployment %q, retrying...", istioName)
201+
return false, nil
202+
}
203+
} else {
204+
t.Logf("Deployment %q still exists, retrying...", istioName)
205+
return false, nil
206+
}
207+
return true, nil
208+
}); err != nil {
209+
return fmt.Errorf("timed out waiting for deployment %v to disappear", istioName)
210+
}
211+
t.Logf("Deleted deployment %q", istioName)
212+
213+
return nil
214+
}
215+
216+
// cleanupGatewayClass deletes the given gatewayclass and waits for the corresponding Istiod deployment to disappear.
217+
func cleanupGatewayClass(t *testing.T, name string) error {
218+
t.Helper()
219+
220+
gc := &gatewayapiv1.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: name}}
221+
if err := kclient.Delete(context.Background(), gc); err != nil && !kerrors.IsNotFound(err) {
222+
return fmt.Errorf("failed to delete gatewayclass %q: %w", name, err)
223+
}
224+
t.Logf("Deleted gatewayclass %q", name)
225+
226+
depl := &appsv1.Deployment{}
227+
istiodName := types.NamespacedName{Namespace: operatorcontroller.DefaultOperandNamespace, Name: "istiod-" + name}
228+
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 1*time.Minute, false, func(context context.Context) (bool, error) {
229+
if err := kclient.Get(context, istiodName, depl); err != nil {
230+
if !kerrors.IsNotFound(err) {
231+
t.Logf("Failed to get deployment %q, retrying...", istiodName)
232+
return false, nil
233+
}
234+
} else {
235+
t.Logf("Deployment %q still exists, retrying...", istiodName)
236+
return false, nil
237+
}
238+
return true, nil
239+
}); err != nil {
240+
return fmt.Errorf("timed out waiting for deployment %v to disappear", istiodName)
241+
}
242+
t.Logf("Deleted deployment %q", istiodName)
243+
244+
return nil
245+
}
246+
247+
// cleanupOLMOperator deletes all components associated with the given OLM operator.
248+
func cleanupOLMOperator(t *testing.T, namespace, name string) error {
249+
operatorName := types.NamespacedName{Namespace: namespace, Name: name}
250+
operator := &olmv1.Operator{}
251+
if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 1*time.Minute, false, func(context context.Context) (bool, error) {
252+
if err := kclient.Get(context, operatorName, operator); err != nil {
253+
t.Logf("Failed to get operator %q, retrying...", operatorName)
254+
return false, nil
255+
}
256+
return true, nil
257+
}); err != nil {
258+
return fmt.Errorf("timed out getting operator %v", operatorName)
259+
}
260+
261+
if operator.Status.Components != nil {
262+
for _, compRef := range operator.Status.Components.Refs {
263+
// OperatorHUB's uninstall action doesn't touch operator's CRDs.
264+
if compRef.Kind != "CustomResourceDefinition" {
265+
unstructuredObj := &unstructured.Unstructured{}
266+
unstructuredObj.SetAPIVersion(compRef.APIVersion)
267+
unstructuredObj.SetKind(compRef.Kind)
268+
unstructuredObj.SetName(compRef.Name)
269+
if len(compRef.Namespace) > 0 {
270+
unstructuredObj.SetNamespace(compRef.Namespace)
271+
}
272+
if err := kclient.Delete(context.Background(), unstructuredObj); err != nil && !kerrors.IsNotFound(err) {
273+
return fmt.Errorf(`failed to delete operator component "%s/%s/%s/%s": %w`, compRef.APIVersion, compRef.Kind, compRef.Namespace, compRef.Name, err)
274+
}
275+
t.Logf(`Deleted operator component "%s/%s/%s/%s"`, compRef.APIVersion, compRef.Kind, compRef.Namespace, compRef.Name)
276+
}
277+
}
278+
} else {
279+
t.Logf("No components found for operator %q", operatorName)
280+
}
281+
282+
return nil
283+
}
284+
183285
// createHttpRoute checks if the HTTPRoute can be created.
184286
// If it can't an error is returned.
185287
func createHttpRoute(namespace, routeName, parentNamespace, hostname, backendRefname string, gateway *gatewayapiv1.Gateway) (*gatewayapiv1.HTTPRoute, error) {

vendor/github.com/operator-framework/api/pkg/operators/v1/doc.go

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vendor/github.com/operator-framework/api/pkg/operators/v1/groupversion_info.go

Lines changed: 28 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)