diff --git a/.gitignore b/.gitignore index d7a932a47..c96681171 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ api7-ingress-controller api7-ingress-controller-conformance-report.yaml *.mdx +.cursor/ + diff --git a/test/conformance/suite_test.go b/test/conformance/suite_test.go index d3c0fa0f6..6ae32ed1b 100644 --- a/test/conformance/suite_test.go +++ b/test/conformance/suite_test.go @@ -1,15 +1,20 @@ package conformance import ( + "context" "fmt" "os" "testing" + "time" "github.com/api7/api7-ingress-controller/test/e2e/framework" "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/retry" + "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) var gatewayClassName = "api7" @@ -24,15 +29,89 @@ spec: controllerName: %s `, gatewayClassName, controllerName) +var gatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: conformance-gateway-proxy + namespace: %s +spec: + statusAddress: + - %s + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: %s +` + +type GatewayProxyOpts struct { + StatusAddress string + AdminKey string + AdminEndpoint string +} + +var defaultGatewayProxyOpts GatewayProxyOpts + +func deleteNamespace(kubectl *k8s.KubectlOptions) { + // gateway api conformance test namespaces + namespacesToDelete := []string{ + "gateway-conformance-infra", + "gateway-conformance-web-backend", + "gateway-conformance-app-backend", + } + + for _, ns := range namespacesToDelete { + _, err := k8s.GetNamespaceE(GinkgoT(), kubectl, ns) + if err == nil { + // Namespace exists, delete it + GinkgoT().Logf("Deleting existing namespace: %s", ns) + err := k8s.DeleteNamespaceE(GinkgoT(), kubectl, ns) + if err != nil { + GinkgoT().Logf("Error deleting namespace %s: %v", ns, err) + continue + } + + // Wait for deletion to complete by checking until GetNamespaceE returns an error + _, err = retry.DoWithRetryE( + GinkgoT(), + fmt.Sprintf("Waiting for namespace %s to be deleted", ns), + 30, + 5*time.Second, + func() (string, error) { + _, err := k8s.GetNamespaceE(GinkgoT(), kubectl, ns) + if err != nil { + // Namespace is gone, which is what we want + return "Namespace deleted", nil + } + return "", fmt.Errorf("namespace %s still exists", ns) + }, + ) + + if err != nil { + GinkgoT().Logf("Error waiting for namespace %s to be deleted: %v", ns, err) + } + } else { + GinkgoT().Logf("Namespace %s does not exist or cannot be accessed", ns) + } + } +} + func TestMain(m *testing.M) { RegisterFailHandler(Fail) f := framework.NewFramework() f.BeforeSuite() - namespace := "api7ee-conformance-test" - + // Check and delete specific namespaces if they exist kubectl := k8s.NewKubectlOptions("", "", "default") + deleteNamespace(kubectl) + + namespace := "api7ee-conformance-test" k8s.KubectlApplyFromString(GinkgoT(), kubectl, gatewayClass) defer k8s.KubectlDeleteFromString(GinkgoT(), kubectl, gatewayClass) @@ -71,9 +150,94 @@ func TestMain(m *testing.M) { StatusAddress: address, }) + defaultGatewayProxyOpts = GatewayProxyOpts{ + StatusAddress: address, + AdminKey: adminKey, + AdminEndpoint: framework.DashboardTLSEndpoint, + } + + patchGatewaysForConformanceTest(context.Background(), f.K8sClient) + code := m.Run() f.AfterSuite() os.Exit(code) } + +func patchGatewaysForConformanceTest(ctx context.Context, k8sClient client.Client) { + var gatewayProxyMap = make(map[string]bool) + + // list all gateways and patch them + patchGateway := func(ctx context.Context, k8sClient client.Client) bool { + gatewayList := &gatewayv1.GatewayList{} + if err := k8sClient.List(ctx, gatewayList); err != nil { + return false + } + + patched := false + for i := range gatewayList.Items { + gateway := &gatewayList.Items[i] + + // check if the gateway already has infrastructure.parametersRef + if gateway.Spec.Infrastructure != nil && + gateway.Spec.Infrastructure.ParametersRef != nil { + continue + } + + GinkgoT().Logf("Patching Gateway %s", gateway.Name) + // check if the gateway proxy has been created, if not, create it + if !gatewayProxyMap[gateway.Namespace] { + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, + gateway.Namespace, + defaultGatewayProxyOpts.StatusAddress, + defaultGatewayProxyOpts.AdminEndpoint, + defaultGatewayProxyOpts.AdminKey) + kubectl := k8s.NewKubectlOptions("", "", gateway.Namespace) + k8s.KubectlApplyFromString(GinkgoT(), kubectl, gatewayProxy) + + // Mark this namespace as having a GatewayProxy + gatewayProxyMap[gateway.Namespace] = true + } + + // add infrastructure.parametersRef + gateway.Spec.Infrastructure = &gatewayv1.GatewayInfrastructure{ + ParametersRef: &gatewayv1.LocalParametersReference{ + Group: "gateway.apisix.io", + Kind: "GatewayProxy", + Name: "conformance-gateway-proxy", + }, + } + + if err := k8sClient.Update(ctx, gateway); err != nil { + GinkgoT().Logf("Failed to patch Gateway %s: %v", gateway.Name, err) + continue + } + + patched = true + GinkgoT().Logf("Successfully patched Gateway %s with GatewayProxy reference", gateway.Name) + } + + return patched + } + + // continuously monitor and patch gateway resources + go func() { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + // clean up the gateway proxy + for namespace := range gatewayProxyMap { + kubectl := k8s.NewKubectlOptions("", "", namespace) + _ = k8s.RunKubectlE(GinkgoT(), kubectl, "delete", "gatewayproxy", "conformance-gateway-proxy") + } + return + case <-ticker.C: + patchGateway(ctx, k8sClient) + } + } + }() +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 43baf0bbb..39f1a0db0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -17,6 +17,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" ) var ( @@ -44,6 +45,7 @@ type Framework struct { kubectlOpts *k8s.KubectlOptions clientset *kubernetes.Clientset restConfig *rest.Config + K8sClient client.Client DB *gorm.DB RawETCD *clientv3.Client @@ -83,6 +85,10 @@ func NewFramework() *Framework { f.GomegaT.Expect(err).ShouldNot(HaveOccurred(), "creating Kubernetes clientset") f.clientset = clientset + k8sClient, err := client.New(restCfg, client.Options{}) + f.GomegaT.Expect(err).ShouldNot(HaveOccurred(), "creating controller-runtime client") + f.K8sClient = k8sClient + _framework = f // BeforeSuite(f.BeforeSuite)