diff --git a/Makefile b/Makefile index 194ae9a0b..067e7b2f1 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,10 @@ GATEAY_API_VERSION ?= v1.2.0 DASHBOARD_VERSION ?= dev TEST_TIMEOUT ?= 45m +APISIX_IMAGE ?= apache/apisix:dev +APISIX_ADMIN_KEY ?= edd1c9f034335f136f87ad84b625c8f1 +APISIX_NAMESPACE ?= apisix-standalone + # CRD Reference Documentation CRD_REF_DOCS_VERSION ?= v0.1.0 CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs @@ -111,6 +115,11 @@ e2e-test: @kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test ./test/e2e/ -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v -ginkgo.focus="$(TEST_FOCUS)" +.PHONY: e2e-test-standalone +e2e-test-standalone: + @kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG + APISIX_IMAGE=$(APISIX_IMAGE) APISIX_ADMIN_KEY=$(APISIX_ADMIN_KEY) APISIX_NAMESPACE=$(APISIX_NAMESPACE) go test ./test/e2e/apisix/ -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v -ginkgo.focus="$(TEST_FOCUS)" + .PHONY: download-api7ee3-chart download-api7ee3-chart: @helm repo add api7 https://charts.api7.ai || true diff --git a/test/e2e/apisix/basic.go b/test/e2e/apisix/basic.go new file mode 100644 index 000000000..32101ba05 --- /dev/null +++ b/test/e2e/apisix/basic.go @@ -0,0 +1,40 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apisix + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var _ = Describe("APISIX Standalone Basic Tests", func() { + s := scaffold.NewScaffold(&scaffold.Options{ + ControllerName: "apisix.apache.org/apisix-ingress-controller", + }) + + Describe("APISIX HTTP Proxy", func() { + It("should handle basic HTTP requests", func() { + httpClient := s.NewAPISIXClient() + Expect(httpClient).NotTo(BeNil()) + + // Test basic connectivity + resp := httpClient.GET("/anything"). + Expect(). + Status(200) + + resp.JSON().Object().ContainsKey("url") + }) + }) +}) diff --git a/test/e2e/apisix/e2e_test.go b/test/e2e/apisix/e2e_test.go new file mode 100644 index 000000000..2b4a84c15 --- /dev/null +++ b/test/e2e/apisix/e2e_test.go @@ -0,0 +1,41 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apisix + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +// TestAPISIXE2E runs e2e tests using the APISIX standalone mode +func TestAPISIXE2E(t *testing.T) { + RegisterFailHandler(Fail) + f := framework.NewAPISIXFramework() + + // init newScaffold function + scaffold.NewScaffold = func(opts *scaffold.Options) scaffold.TestScaffold { + return scaffold.NewAPISIXScaffold(opts) + } + + BeforeSuite(f.BeforeSuite) + AfterSuite(f.AfterSuite) + + _, _ = fmt.Fprintf(GinkgoWriter, "Starting APISIX standalone e2e suite\n") + RunSpecs(t, "apisix standalone e2e suite") +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6d80833c6..57e1628ab 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -24,6 +24,7 @@ import ( "github.com/apache/apisix-ingress-controller/test/e2e/framework" _ "github.com/apache/apisix-ingress-controller/test/e2e/gatewayapi" _ "github.com/apache/apisix-ingress-controller/test/e2e/ingress" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) // Run e2e tests using the Ginkgo runner. @@ -31,6 +32,11 @@ func TestE2E(t *testing.T) { RegisterFailHandler(Fail) f := framework.NewFramework() + // init newScaffold function + scaffold.NewScaffold = func(opts *scaffold.Options) scaffold.TestScaffold { + return scaffold.NewAPI7Scaffold(opts) + } + BeforeSuite(f.BeforeSuite) AfterSuite(f.AfterSuite) diff --git a/test/e2e/framework/apisix.go b/test/e2e/framework/apisix.go new file mode 100644 index 000000000..a5db6d7f1 --- /dev/null +++ b/test/e2e/framework/apisix.go @@ -0,0 +1,156 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "os" + "text/template" + "time" + + "github.com/Masterminds/sprig/v3" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/testing" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +//go:embed manifests/apisix-standalone.yaml +var apisixStandaloneTemplate string + +// APISIXDeployOptions contains options for APISIX standalone deployment +type APISIXDeployOptions struct { + Namespace string + Image string + AdminKey string + ServiceType string + HTTPNodePort int32 + HTTPSNodePort int32 + AdminNodePort int32 +} + +// APISIXDeployer implements DataPlaneDeployer for APISIX standalone +type APISIXDeployer struct { + kubectlOpts *k8s.KubectlOptions + opts *APISIXDeployOptions + service *corev1.Service + t testing.TestingT +} + +// NewAPISIXDeployer creates a new APISIX deployer +func NewAPISIXDeployer(t testing.TestingT, kubectlOpts *k8s.KubectlOptions, opts *APISIXDeployOptions) *APISIXDeployer { + if opts.Image == "" { + opts.Image = getEnvOrDefault("APISIX_IMAGE", "apache/apisix:dev") + } + if opts.AdminKey == "" { + opts.AdminKey = getEnvOrDefault("APISIX_ADMIN_KEY", "edd1c9f034335f136f87ad84b625c8f1") + } + if opts.Namespace == "" { + opts.Namespace = getEnvOrDefault("APISIX_NAMESPACE", "apisix-standalone") + } + if opts.ServiceType == "" { + opts.ServiceType = "ClusterIP" + } + + return &APISIXDeployer{ + kubectlOpts: kubectlOpts, + opts: opts, + t: t, + } +} + +func (d *APISIXDeployer) GetService() *corev1.Service { + return d.service +} + +// Deploy deploys APISIX standalone +func (d *APISIXDeployer) Deploy(ctx context.Context) error { + // Parse and execute template + tmpl, err := template.New("apisix-standalone").Funcs(sprig.TxtFuncMap()).Parse(apisixStandaloneTemplate) + if err != nil { + return fmt.Errorf("failed to parse template: %w", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, d.opts); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + // Apply the manifest + if err := k8s.KubectlApplyFromStringE(d.t, d.kubectlOpts, buf.String()); err != nil { + return fmt.Errorf("failed to apply APISIX manifest: %w", err) + } + + // Wait for deployment to be ready + if err := d.waitForDeployment(ctx); err != nil { + return fmt.Errorf("failed to wait for deployment: %w", err) + } + + // Get service + service, err := k8s.GetServiceE(d.t, d.kubectlOpts, "apisix") + if err != nil { + return fmt.Errorf("failed to get APISIX service: %w", err) + } + d.service = service + + return nil +} + +// Cleanup removes APISIX standalone deployment +func (d *APISIXDeployer) Cleanup(ctx context.Context) error { + // Delete namespace which will clean up all resources + return k8s.DeleteNamespaceE(d.t, d.kubectlOpts, d.opts.Namespace) +} + +// waitForDeployment waits for the APISIX deployment to be ready +func (d *APISIXDeployer) waitForDeployment(ctx context.Context) error { + return wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { + pods, err := k8s.ListPodsE(d.t, d.kubectlOpts, metav1.ListOptions{ + LabelSelector: "app=apisix", + }) + if err != nil { + return false, err + } + + if len(pods) == 0 { + return false, nil + } + + for _, pod := range pods { + if pod.Status.Phase != corev1.PodRunning { + return false, nil + } + + // Check if all containers are ready + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status != corev1.ConditionTrue { + return false, nil + } + } + } + + return true, nil + }) +} + +// getEnvOrDefault returns environment variable value or default +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} diff --git a/test/e2e/framework/apisix_framework.go b/test/e2e/framework/apisix_framework.go new file mode 100644 index 000000000..c3721d36f --- /dev/null +++ b/test/e2e/framework/apisix_framework.go @@ -0,0 +1,206 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/logger" + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + _apisixNamespace = "apisix-standalone-e2e" + _apisixFramework *APISIXFramework +) + +// APISIXFramework implements TestFramework for APISIX standalone +type APISIXFramework struct { + Context context.Context + GinkgoT GinkgoTInterface + GomegaT *GomegaWithT + + Logger logger.TestLogger + + kubectlOpts *k8s.KubectlOptions + clientset *kubernetes.Clientset + restConfig *rest.Config + K8sClient client.Client + namespace string +} + +// NewAPISIXFramework creates a new APISIX framework +func NewAPISIXFramework() *APISIXFramework { + f := &APISIXFramework{ + GinkgoT: GinkgoT(), + GomegaT: NewWithT(GinkgoT(4)), + Logger: logger.Terratest, + } + + f.Context = context.TODO() + + // Use environment variable for namespace if set + namespace := os.Getenv("APISIX_NAMESPACE") + if namespace == "" { + namespace = _apisixNamespace + } + + f.namespace = namespace + + f.kubectlOpts = k8s.NewKubectlOptions("", "", namespace) + restCfg, err := buildRestConfig("") + f.GomegaT.Expect(err).ShouldNot(HaveOccurred(), "building API Server rest config") + f.restConfig = restCfg + + clientset, err := kubernetes.NewForConfig(restCfg) + f.GomegaT.Expect(err).ShouldNot(HaveOccurred(), "creating Kubernetes clientset") + f.clientset = clientset + + k8sClient, err := client.New(restCfg, client.Options{}) + f.GomegaT.Expect(err).ShouldNot(HaveOccurred(), "creating controller-runtime client") + f.K8sClient = k8sClient + + _apisixFramework = f + + return f +} + +// BeforeSuite initializes the APISIX test environment +func (f *APISIXFramework) BeforeSuite() { + f.Logf("Starting APISIX standalone test suite") + _ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, f.namespace) + + Eventually(func() error { + _, err := k8s.GetNamespaceE(GinkgoT(), f.kubectlOpts, f.namespace) + if k8serrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("namespace %s still exists", f.namespace) + }, "1m", "2s").Should(Succeed()) + + k8s.CreateNamespace(GinkgoT(), f.kubectlOpts, f.namespace) + + f.Logf("APISIX standalone test environment initialized") +} + +// AfterSuite cleans up the APISIX test environment +func (f *APISIXFramework) AfterSuite() { + f.Logf("Cleaning up APISIX standalone test environment") + + // Clean up namespace + _ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, f.kubectlOpts.Namespace) +} + +// GetFramework returns the global APISIX framework instance +func GetAPISIXFramework() *APISIXFramework { + return _apisixFramework +} + +// Logf logs a formatted message +func (f *APISIXFramework) Logf(format string, v ...any) { + f.Logger.Logf(f.GinkgoT, format, v...) +} + +func (f *APISIXFramework) DeployIngress(opts IngressDeployOpts) { + buf := bytes.NewBuffer(nil) + + err := IngressSpecTpl.Execute(buf, opts) + f.GomegaT.Expect(err).ToNot(HaveOccurred(), "rendering ingress spec") + + kubectlOpts := k8s.NewKubectlOptions("", "", opts.Namespace) + + k8s.KubectlApplyFromString(f.GinkgoT, kubectlOpts, buf.String()) + + err = WaitPodsAvailable(f.GinkgoT, kubectlOpts, metav1.ListOptions{ + LabelSelector: "control-plane=controller-manager", + }) + f.GomegaT.Expect(err).ToNot(HaveOccurred(), "waiting for controller-manager pod ready") + f.WaitControllerManagerLog("All cache synced successfully", 0, time.Minute) +} + +func (f *APISIXFramework) WaitControllerManagerLog(keyword string, sinceSeconds int64, timeout time.Duration) { + f.WaitPodsLog("control-plane=controller-manager", keyword, sinceSeconds, timeout) +} + +func (f *APISIXFramework) WaitDPLog(keyword string, sinceSeconds int64, timeout time.Duration) { + f.WaitPodsLog("app.kubernetes.io/name=apisix", keyword, sinceSeconds, timeout) +} + +func (f *APISIXFramework) WaitPodsLog(selector, keyword string, sinceSeconds int64, timeout time.Duration) { + pods := f.ListRunningPods(selector) + wg := sync.WaitGroup{} + for _, p := range pods { + wg.Add(1) + go func(p corev1.Pod) { + defer wg.Done() + opts := corev1.PodLogOptions{Follow: true} + if sinceSeconds > 0 { + opts.SinceSeconds = ptr.To(sinceSeconds) + } else { + opts.TailLines = ptr.To(int64(0)) + } + logStream, err := f.clientset.CoreV1().Pods(p.Namespace).GetLogs(p.Name, &opts).Stream(context.Background()) + f.GomegaT.Expect(err).Should(gomega.BeNil()) + scanner := bufio.NewScanner(logStream) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, keyword) { + return + } + } + }(p) + } + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + select { + case <-c: + return + case <-time.After(timeout): + f.GinkgoT.Error("wait log timeout") + } +} + +func (f *APISIXFramework) ListRunningPods(selector string) []corev1.Pod { + pods, err := f.clientset.CoreV1().Pods(f.namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: selector, + }) + f.GomegaT.Expect(err).ShouldNot(gomega.HaveOccurred(), "list pod: ", selector) + runningPods := make([]corev1.Pod, 0) + for _, p := range pods.Items { + if p.Status.Phase == corev1.PodRunning && p.DeletionTimestamp == nil { + runningPods = append(runningPods, p) + } + } + return runningPods +} diff --git a/test/e2e/framework/dashboard.go b/test/e2e/framework/dashboard.go index d3006a6da..b6b4c43e4 100644 --- a/test/e2e/framework/dashboard.go +++ b/test/e2e/framework/dashboard.go @@ -43,16 +43,6 @@ var ( ) func init() { - API7EELicense = os.Getenv("API7_EE_LICENSE") - if API7EELicense == "" { - panic("env {API7_EE_LICENSE} is required") - } - - dashboardVersion = os.Getenv("DASHBOARD_VERSION") - if dashboardVersion == "" { - dashboardVersion = "dev" - } - tmpl, err := template.New("values.yaml").Parse(` dashboard: image: diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 4a481987d..d1da145ba 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -18,6 +18,7 @@ import ( _ "embed" "encoding/base64" "fmt" + "os" "time" "github.com/gruntwork-io/terratest/modules/k8s" @@ -76,6 +77,16 @@ type Framework struct { // NewFramework create a global framework with special settings. func NewFramework() *Framework { + API7EELicense = os.Getenv("API7_EE_LICENSE") + if API7EELicense == "" { + panic("env {API7_EE_LICENSE} is required") + } + + dashboardVersion = os.Getenv("DASHBOARD_VERSION") + if dashboardVersion == "" { + dashboardVersion = "dev" + } + f := &Framework{ GinkgoT: GinkgoT(), GomegaT: NewWithT(GinkgoT(4)), diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml new file mode 100644 index 000000000..11a43fac8 --- /dev/null +++ b/test/e2e/framework/manifests/apisix-standalone.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: apisix-conf + namespace: {{ .Namespace }} +data: + config.yaml: | + apisix: + enable_admin: true + admin_key: + - name: admin + key: {{ .AdminKey }} + role: admin + ssl: + enabled: true + nginx_config: + worker_processes: 2 + error_log_level: info + deployment: + role: traditional + role_traditional: + config_provider: yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: apisix + namespace: {{ .Namespace }} + labels: + app: apisix +spec: + replicas: 1 + selector: + matchLabels: + app: apisix + template: + metadata: + labels: + app: apisix + spec: + containers: + - name: apisix + image: {{ .Image }} + ports: + - name: http + containerPort: 9080 + protocol: TCP + - name: https + containerPort: 9443 + protocol: TCP + - name: admin + containerPort: 9180 + protocol: TCP + volumeMounts: + - name: conf + mountPath: /usr/local/apisix/conf/config.yaml + subPath: config.yaml + volumes: + - name: conf + configMap: + name: apisix-conf +--- +apiVersion: v1 +kind: Service +metadata: + name: apisix + namespace: {{ .Namespace }} + labels: + app: apisix +spec: + type: {{ .ServiceType | default "NodePort" }} + ports: + - port: 9080 + name: http + protocol: TCP + targetPort: 9080 + - port: 9443 + name: https + protocol: TCP + targetPort: 9443 + - port: 9180 + name: admin + protocol: TCP + targetPort: 9180 + selector: + app: apisix diff --git a/test/e2e/gatewayapi/controller.go b/test/e2e/gatewayapi/controller.go index 5ded0be5a..d6840e3da 100644 --- a/test/e2e/gatewayapi/controller.go +++ b/test/e2e/gatewayapi/controller.go @@ -72,7 +72,7 @@ spec: name: apisix-proxy-config ` - var ResourceApplied = func(s *scaffold.Scaffold, resourType, resourceName, ns, resourceRaw string, observedGeneration int) { + var ResourceApplied = func(s scaffold.TestScaffold, resourType, resourceName, ns, resourceRaw string, observedGeneration int) { Expect(s.CreateResourceFromStringWithNamespace(resourceRaw, ns)). NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourType)) @@ -90,7 +90,7 @@ spec: ) time.Sleep(1 * time.Second) } - var beforeEach = func(s *scaffold.Scaffold, gatewayName string) { + var beforeEach = func(s scaffold.TestScaffold, gatewayName string) { err := s.CreateResourceFromString(fmt.Sprintf(` kind: Namespace apiVersion: v1 @@ -169,7 +169,7 @@ spec: }) It("Apply resource ", func() { ResourceApplied(s1, "HTTPRoute", "httpbin", "gateway1", route1, 1) - routes, err := s1.DefaultDataplaneResource().Route().List(s1.Context) + routes, err := s1.DefaultDataplaneResource().Route().List(s1.GetContext()) Expect(err).NotTo(HaveOccurred()) Expect(routes).To(HaveLen(1)) assert.Equal(GinkgoT(), routes[0].Labels["k8s/controller-name"], "apisix.apache.org/apisix-ingress-controller-1") @@ -215,7 +215,7 @@ spec: }) It("Apply resource ", func() { ResourceApplied(s2, "HTTPRoute", "httpbin2", "gateway2", route2, 1) - routes, err := s2.DefaultDataplaneResource().Route().List(s2.Context) + routes, err := s2.DefaultDataplaneResource().Route().List(s2.GetContext()) Expect(err).NotTo(HaveOccurred()) Expect(routes).To(HaveLen(1)) assert.Equal(GinkgoT(), routes[0].Labels["k8s/controller-name"], "apisix.apache.org/apisix-ingress-controller-2") diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index 39137eed9..0424c7595 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -32,7 +32,7 @@ var Cert = strings.TrimSpace(framework.TestServerCert) var Key = strings.TrimSpace(framework.TestServerKey) -func createSecret(s *scaffold.Scaffold, secretName string) { +func createSecret(s scaffold.TestScaffold, secretName string) { err := s.NewKubeTlsSecret(secretName, Cert, Key) assert.Nil(GinkgoT(), err, "create secret error") } diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index d710875a3..2f77034aa 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -881,7 +881,7 @@ spec: return err.Error() }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(ContainSubstring(`httproutepolicies.apisix.apache.org "http-route-policy-0" not found`)) // access the route without additional vars should be OK - message := retry.DoWithRetry(s.GinkgoT, "", 10, time.Second, func() (string, error) { + message := retry.DoWithRetry(s.GetGinkgoT(), "", 10, time.Second, func() (string, error) { statusCode := s.NewAPISIXClient(). GET("/get"). WithHost("httpbin.example"). @@ -892,7 +892,7 @@ spec: } return "request OK", nil }) - s.Logf(message) + s.GetGinkgoT().Logf(message) }) It("HTTPRoutePolicy conflicts", func() { @@ -975,7 +975,7 @@ spec: err := s.CreateResourceFromString(spec) Expect(err).NotTo(HaveOccurred(), "creating HTTPRoutePolicy") // wait for HTTPRoutePolicy is Accepted - framework.HTTPRoutePolicyMustHaveCondition(s.GinkgoT, s.K8sClient, 10*time.Second, + framework.HTTPRoutePolicyMustHaveCondition(s.GetGinkgoT(), s.GetK8sClient(), 10*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, types.NamespacedName{Namespace: s.Namespace(), Name: name}, metav1.Condition{ @@ -984,7 +984,7 @@ spec: ) } for _, name := range []string{"http-route-policy-0", "http-route-policy-1", "http-route-policy-2"} { - framework.HTTPRoutePolicyMustHaveCondition(s.GinkgoT, s.K8sClient, 10*time.Second, + framework.HTTPRoutePolicyMustHaveCondition(s.GetGinkgoT(), s.GetK8sClient(), 10*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, types.NamespacedName{Namespace: s.Namespace(), Name: name}, metav1.Condition{ @@ -1008,7 +1008,7 @@ spec: err := s.DeleteResource("HTTPRoutePolicy", "http-route-policy-2") Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoutePolicy %s", "http-route-policy-2") for _, name := range []string{"http-route-policy-0", "http-route-policy-1"} { - framework.HTTPRoutePolicyMustHaveCondition(s.GinkgoT, s.K8sClient, 10*time.Second, + framework.HTTPRoutePolicyMustHaveCondition(s.GetGinkgoT(), s.GetK8sClient(), 10*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, types.NamespacedName{Namespace: s.Namespace(), Name: name}, metav1.Condition{ @@ -1029,7 +1029,7 @@ spec: By("update HTTPRoutePolicy") err = s.CreateResourceFromString(httpRoutePolicy1Priority20) Expect(err).NotTo(HaveOccurred(), "update HTTPRoutePolicy's priority to 20") - framework.HTTPRoutePolicyMustHaveCondition(s.GinkgoT, s.K8sClient, 10*time.Second, + framework.HTTPRoutePolicyMustHaveCondition(s.GetGinkgoT(), s.GetK8sClient(), 10*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, types.NamespacedName{Namespace: s.Namespace(), Name: "http-route-policy-1"}, metav1.Condition{ @@ -1037,7 +1037,7 @@ spec: }, ) for _, name := range []string{"http-route-policy-0", "http-route-policy-1"} { - framework.HTTPRoutePolicyMustHaveCondition(s.GinkgoT, s.K8sClient, 10*time.Second, + framework.HTTPRoutePolicyMustHaveCondition(s.GetGinkgoT(), s.GetK8sClient(), 10*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, types.NamespacedName{Namespace: s.Namespace(), Name: name}, metav1.Condition{ @@ -1089,7 +1089,7 @@ spec: By("delete the HTTPRoute, assert the HTTPRoutePolicy's status will be changed") err := s.DeleteResource("HTTPRoute", "httpbin") Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute") - message := retry.DoWithRetry(s.GinkgoT, "request the deleted route", 10, time.Second, func() (string, error) { + message := retry.DoWithRetry(s.GetGinkgoT(), "request the deleted route", 10, time.Second, func() (string, error) { statusCode := s.NewAPISIXClient(). GET("/get"). WithHost("httpbin.example"). @@ -1102,7 +1102,7 @@ spec: } return "the route is deleted", nil }) - s.Logf(message) + s.GetGinkgoT().Logf(message) Eventually(func() string { spec, err := s.GetResourceYaml("HTTPRoutePolicy", "http-route-policy-0") diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index fb0de348a..a5a879dc8 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -38,7 +38,7 @@ var Cert = strings.TrimSpace(framework.TestServerCert) var Key = strings.TrimSpace(framework.TestServerKey) -func createSecret(s *scaffold.Scaffold, secretName string) { +func createSecret(s scaffold.TestScaffold, secretName string) { err := s.NewKubeTlsSecret(secretName, Cert, Key) assert.Nil(GinkgoT(), err, "create secret error") } @@ -734,7 +734,7 @@ spec: By("create HTTPRoutePolicy") err = s.CreateResourceFromString(httpRoutePolicySpec0) Expect(err).NotTo(HaveOccurred(), "creating HTTPRoutePolicy") - framework.HTTPRoutePolicyMustHaveCondition(s.GinkgoT, s.K8sClient, 8*time.Second, + framework.HTTPRoutePolicyMustHaveCondition(s.GetGinkgoT(), s.GetK8sClient(), 8*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "apisix"}, types.NamespacedName{Namespace: s.Namespace(), Name: "http-route-policy-0"}, metav1.Condition{ @@ -748,7 +748,7 @@ spec: err = s.DeleteResource("Ingress", "default") Expect(err).NotTo(HaveOccurred(), "delete Ingress") - err = framework.EventuallyHTTPRoutePolicyHaveStatus(s.K8sClient, 8*time.Second, + err = framework.EventuallyHTTPRoutePolicyHaveStatus(s.GetK8sClient(), 8*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "http-route-policy-0"}, func(_ v1alpha1.HTTPRoutePolicy, status v1alpha1.PolicyStatus) bool { return len(status.Ancestors) == 0 diff --git a/test/e2e/scaffold/api7_deployer.go b/test/e2e/scaffold/api7_deployer.go new file mode 100644 index 000000000..019fe7466 --- /dev/null +++ b/test/e2e/scaffold/api7_deployer.go @@ -0,0 +1,244 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + + "github.com/gavv/httpexpect/v2" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/testing" + . "github.com/onsi/ginkgo/v2" + + "github.com/apache/apisix-ingress-controller/pkg/dashboard" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" +) + +// API7Deployer implements Deployer interface for API7 enterprise version +type API7Deployer struct { + t testing.TestingT + kubectlOpts *k8s.KubectlOptions + framework *framework.Framework + opts *DeployerOptions + + // API7-specific resources + apisixCli dashboard.Dashboard + httpTunnel *k8s.Tunnel + httpsTunnel *k8s.Tunnel +} + +// NewAPI7Deployer creates a new API7 deployer +func NewAPI7Deployer(t testing.TestingT, kubectlOpts *k8s.KubectlOptions, framework *framework.Framework, opts *DeployerOptions) (Deployer, error) { + if opts == nil { + return nil, fmt.Errorf("deployer options cannot be nil") + } + + return &API7Deployer{ + t: t, + kubectlOpts: kubectlOpts, + framework: framework, + opts: opts, + }, nil +} + +// Deploy deploys API7 dashboard and gateway +func (d *API7Deployer) Deploy(ctx context.Context) error { + // Deploy API7 dashboard and gateway + // This will use the existing framework logic + d.framework.DeployComponents() + + // Create tunnels for gateway access + d.createTunnels() + + // Initialize data plane client + return d.initDataPlaneClient() +} + +// Cleanup cleans up API7 resources +func (d *API7Deployer) Cleanup(ctx context.Context) error { + // Close tunnels + if d.httpTunnel != nil { + d.httpTunnel.Close() + } + if d.httpsTunnel != nil { + d.httpsTunnel.Close() + } + + // Cleanup API7 dashboard and gateway resources + // The framework cleanup will be handled by namespace deletion + return nil +} + +// GetHTTPClient returns HTTP client for API7 gateway +func (d *API7Deployer) GetHTTPClient() *httpexpect.Expect { + if d.httpTunnel == nil { + // Create tunnel if not exists + d.createTunnels() + } + + u := url.URL{ + Scheme: "http", + Host: d.httpTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// GetHTTPSClient returns HTTPS client for API7 gateway +func (d *API7Deployer) GetHTTPSClient(host string) *httpexpect.Expect { + if d.httpsTunnel == nil { + // Create tunnel if not exists + d.createTunnels() + } + + return d.newHTTPSClientWithCertificates(host, true, nil, nil) +} + +// GetAdminClient returns admin client for API7 dashboard +func (d *API7Deployer) GetAdminClient() *httpexpect.Expect { + // Return a new HTTP client pointing to the dashboard endpoint + dashboardEndpoint := d.framework.GetDashboardEndpoint() + u := url.URL{ + Scheme: "http", + Host: dashboardEndpoint, + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// GetAdminKey returns admin key for API7 +func (d *API7Deployer) GetAdminKey() string { + return d.opts.AdminKey +} + +// GetDataplaneResource returns dashboard cluster for admin operations +func (d *API7Deployer) GetDataplaneResource() dashboard.Cluster { + return d.apisixCli.Cluster("default") +} + +// GetDataplaneResourceHTTPS returns HTTPS dashboard cluster +func (d *API7Deployer) GetDataplaneResourceHTTPS() dashboard.Cluster { + return d.apisixCli.Cluster("default-https") +} + +// GetMode returns the deployment mode +func (d *API7Deployer) GetMode() DeployMode { + return DeployModeAPI7 +} + +// initDataPlaneClient initializes the dashboard client +func (d *API7Deployer) initDataPlaneClient() error { + var err error + d.apisixCli, err = dashboard.NewClient() + if err != nil { + return fmt.Errorf("creating apisix client: %w", err) + } + + url := fmt.Sprintf("http://%s/apisix/admin", d.framework.GetDashboardEndpoint()) + + err = d.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ + Name: "default", + ControllerName: d.opts.ControllerName, + Labels: map[string]string{"k8s/controller-name": d.opts.ControllerName}, + BaseURL: url, + AdminKey: d.opts.AdminKey, + }) + if err != nil { + return fmt.Errorf("adding cluster: %w", err) + } + + httpsURL := fmt.Sprintf("https://%s/apisix/admin", d.framework.GetDashboardEndpointHTTPS()) + err = d.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ + Name: "default-https", + BaseURL: httpsURL, + AdminKey: d.opts.AdminKey, + SkipTLSVerify: true, + }) + if err != nil { + return fmt.Errorf("adding https cluster: %w", err) + } + + return nil +} + +// createTunnels creates HTTP and HTTPS tunnels for API7 gateway +func (d *API7Deployer) createTunnels() { + // API7 uses a fixed gateway service name pattern + gatewayServiceName := "api7ee3-apisix-gateway" + + // Standard API7 gateway ports + gatewayHTTPPort := 80 + gatewayHTTPSPort := 443 + + // Create HTTP tunnel + if d.httpTunnel == nil { + httpTunnel := k8s.NewTunnel(d.kubectlOpts, k8s.ResourceTypeService, gatewayServiceName, 0, gatewayHTTPPort) + httpTunnel.ForwardPort(d.t) + d.httpTunnel = httpTunnel + } + + // Create HTTPS tunnel + if d.httpsTunnel == nil { + httpsTunnel := k8s.NewTunnel(d.kubectlOpts, k8s.ResourceTypeService, gatewayServiceName, 0, gatewayHTTPSPort) + httpsTunnel.ForwardPort(d.t) + d.httpsTunnel = httpsTunnel + } +} + +// newHTTPSClientWithCertificates creates HTTPS client with certificates +func (d *API7Deployer) newHTTPSClientWithCertificates( + host string, insecure bool, ca *x509.CertPool, certs []tls.Certificate, +) *httpexpect.Expect { + u := url.URL{ + Scheme: "https", + Host: d.httpsTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecure, + ServerName: host, + RootCAs: ca, + Certificates: certs, + }, + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/api7_scaffold.go similarity index 93% rename from test/e2e/scaffold/scaffold.go rename to test/e2e/scaffold/api7_scaffold.go index 9d7f72ea3..2cefe8d8a 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/api7_scaffold.go @@ -17,7 +17,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "net" "net/http" "net/url" "os" @@ -34,6 +33,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/apache/apisix-ingress-controller/pkg/dashboard" "github.com/apache/apisix-ingress-controller/pkg/utils" @@ -86,16 +86,19 @@ type Scaffold struct { apisixCli dashboard.Dashboard - gatewaygroupid string - apisixHttpTunnel *k8s.Tunnel - apisixHttpsTunnel *k8s.Tunnel - apisixTCPTunnel *k8s.Tunnel - apisixTLSOverTCPTunnel *k8s.Tunnel - apisixUDPTunnel *k8s.Tunnel - // apisixControlTunnel *k8s.Tunnel + gatewaygroupid string + apisixHttpTunnel *k8s.Tunnel + apisixHttpsTunnel *k8s.Tunnel // Support for multiple Gateway groups additionalGatewayGroups map[string]*GatewayGroupResources + + // Deployer interface for data plane deployment + deployer Deployer +} + +func (s *Scaffold) DeployNginx(options framework.NginxOptions) { + s.Framework.DeployNginx(options) } // GatewayGroupResources contains resources associated with a specific Gateway group @@ -133,7 +136,7 @@ func GetKubeconfig() string { } // NewScaffold creates an e2e test scaffold. -func NewScaffold(o *Options) *Scaffold { +func NewAPI7Scaffold(o *Options) *Scaffold { if o.Name == "" { o.Name = "default" } @@ -174,7 +177,7 @@ func NewScaffold(o *Options) *Scaffold { // NewDefaultScaffold creates a scaffold with some default options. // apisix-version default v2 -func NewDefaultScaffold() *Scaffold { +func NewDefaultScaffold() TestScaffold { return NewScaffold(&Options{}) } @@ -227,45 +230,6 @@ func (s *Scaffold) GetAPISIXHTTPSEndpoint() string { return s.apisixHttpsTunnel.Endpoint() } -// NewAPISIXClientWithTCPProxy creates the HTTP client but with the TCP proxy of APISIX. -func (s *Scaffold) NewAPISIXClientWithTCPProxy() *httpexpect.Expect { - u := url.URL{ - Scheme: "http", - Host: s.apisixTCPTunnel.Endpoint(), - } - return httpexpect.WithConfig(httpexpect.Config{ - BaseURL: u.String(), - Client: &http.Client{ - Transport: &http.Transport{}, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - }, - Reporter: httpexpect.NewAssertReporter( - httpexpect.NewAssertReporter(GinkgoT()), - ), - }) -} - -func (s *Scaffold) DNSResolver() *net.Resolver { - return &net.Resolver{ - PreferGo: false, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{ - Timeout: time.Millisecond * time.Duration(10000), - } - return d.DialContext(ctx, "udp", s.apisixUDPTunnel.Endpoint()) - }, - } -} - -func (s *Scaffold) DialTLSOverTcp(serverName string) (*tls.Conn, error) { - return tls.Dial("tcp", s.apisixTLSOverTCPTunnel.Endpoint(), &tls.Config{ - InsecureSkipVerify: true, - ServerName: serverName, - }) -} - func (s *Scaffold) UpdateNamespace(ns string) { s.kubectlOptions.Namespace = ns } @@ -812,3 +776,20 @@ func (s *Scaffold) GetGatewayGroupHTTPSEndpoint(gatewayGroupID string) (string, func (s *Scaffold) CurrentGatewayGroupID() string { return s.gatewaygroupid } + +func (s *Scaffold) GetContext() context.Context { + return s.Context +} + +func (s *Scaffold) GetGinkgoT() GinkgoTInterface { + return s.GinkgoT +} + +func (s *Scaffold) GetK8sClient() client.Client { + return s.K8sClient +} + +// GetDeployer returns the underlying deployer instance +func (s *Scaffold) GetDeployer() Deployer { + return s.deployer +} diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go new file mode 100644 index 000000000..93f74c738 --- /dev/null +++ b/test/e2e/scaffold/apisix_deployer.go @@ -0,0 +1,504 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "net/url" + + "github.com/gavv/httpexpect/v2" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/testing" + . "github.com/onsi/ginkgo/v2" + corev1 "k8s.io/api/core/v1" + + v1 "github.com/apache/apisix-ingress-controller/api/dashboard/v1" + "github.com/apache/apisix-ingress-controller/pkg/dashboard" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" +) + +// VirtualCluster implements dashboard.Cluster interface for APISIX standalone mode +// It provides a virtual cluster that talks directly to APISIX admin API +// For now, it's a minimal implementation that returns "not supported" errors +type VirtualCluster struct { + name string + adminClient *httpexpect.Expect + adminKey string + baseURL string + httpsMode bool +} + +// VirtualCluster methods that implement dashboard.Cluster interface +func (v *VirtualCluster) Route() dashboard.Route { + return &unsupportedRoute{} +} + +func (v *VirtualCluster) Service() dashboard.Service { + return &unsupportedService{} +} + +func (v *VirtualCluster) SSL() dashboard.SSL { + return &unsupportedSSL{} +} + +func (v *VirtualCluster) StreamRoute() dashboard.StreamRoute { + return &unsupportedStreamRoute{} +} + +func (v *VirtualCluster) GlobalRule() dashboard.GlobalRule { + return &unsupportedGlobalRule{} +} + +func (v *VirtualCluster) Consumer() dashboard.Consumer { + return &unsupportedConsumer{} +} + +func (v *VirtualCluster) Plugin() dashboard.Plugin { + return &unsupportedPlugin{} +} + +func (v *VirtualCluster) PluginConfig() dashboard.PluginConfig { + return &unsupportedPluginConfig{} +} + +func (v *VirtualCluster) Schema() dashboard.Schema { + return &unsupportedSchema{} +} + +func (v *VirtualCluster) PluginMetadata() dashboard.PluginMetadata { + return &unsupportedPluginMetadata{} +} + +func (v *VirtualCluster) Validator() dashboard.APISIXSchemaValidator { + return &unsupportedValidator{} +} + +func (v *VirtualCluster) String() string { + return fmt.Sprintf("virtual cluster %s", v.name) +} + +func (v *VirtualCluster) HasSynced(ctx context.Context) error { + // For APISIX standalone, we always consider it synced + return nil +} + +func (v *VirtualCluster) HealthCheck(ctx context.Context) error { + // Simple health check via admin API + resp := v.adminClient.GET("/apisix/admin"). + WithHeader("X-API-KEY", v.adminKey). + Expect() + if resp.Raw().StatusCode >= 200 && resp.Raw().StatusCode < 300 { + return nil + } + return fmt.Errorf("APISIX admin API health check failed with status %d", resp.Raw().StatusCode) +} + +// Unsupported implementations that return "not supported" errors +// These can be implemented later if specific tests require them + +type unsupportedRoute struct{} + +func (u *unsupportedRoute) Get(ctx context.Context, name string) (*v1.Route, error) { + return nil, fmt.Errorf("route operations not supported in virtual cluster") +} +func (u *unsupportedRoute) List(ctx context.Context, args ...any) ([]*v1.Route, error) { + return nil, fmt.Errorf("route operations not supported in virtual cluster") +} +func (u *unsupportedRoute) Create(ctx context.Context, route *v1.Route) (*v1.Route, error) { + return nil, fmt.Errorf("route operations not supported in virtual cluster") +} +func (u *unsupportedRoute) Delete(ctx context.Context, route *v1.Route) error { + return fmt.Errorf("route operations not supported in virtual cluster") +} +func (u *unsupportedRoute) Update(ctx context.Context, route *v1.Route) (*v1.Route, error) { + return nil, fmt.Errorf("route operations not supported in virtual cluster") +} + +type unsupportedService struct{} + +func (u *unsupportedService) Get(ctx context.Context, name string) (*v1.Service, error) { + return nil, fmt.Errorf("service operations not supported in virtual cluster") +} +func (u *unsupportedService) List(ctx context.Context, args ...any) ([]*v1.Service, error) { + return nil, fmt.Errorf("service operations not supported in virtual cluster") +} +func (u *unsupportedService) Create(ctx context.Context, svc *v1.Service) (*v1.Service, error) { + return nil, fmt.Errorf("service operations not supported in virtual cluster") +} +func (u *unsupportedService) Delete(ctx context.Context, svc *v1.Service) error { + return fmt.Errorf("service operations not supported in virtual cluster") +} +func (u *unsupportedService) Update(ctx context.Context, svc *v1.Service) (*v1.Service, error) { + return nil, fmt.Errorf("service operations not supported in virtual cluster") +} + +type unsupportedSSL struct{} + +func (u *unsupportedSSL) Get(ctx context.Context, name string) (*v1.Ssl, error) { + return nil, fmt.Errorf("SSL operations not supported in virtual cluster") +} +func (u *unsupportedSSL) List(ctx context.Context, args ...any) ([]*v1.Ssl, error) { + return nil, fmt.Errorf("SSL operations not supported in virtual cluster") +} +func (u *unsupportedSSL) Create(ctx context.Context, ssl *v1.Ssl) (*v1.Ssl, error) { + return nil, fmt.Errorf("SSL operations not supported in virtual cluster") +} +func (u *unsupportedSSL) Delete(ctx context.Context, ssl *v1.Ssl) error { + return fmt.Errorf("SSL operations not supported in virtual cluster") +} +func (u *unsupportedSSL) Update(ctx context.Context, ssl *v1.Ssl) (*v1.Ssl, error) { + return nil, fmt.Errorf("SSL operations not supported in virtual cluster") +} + +type unsupportedStreamRoute struct{} + +func (u *unsupportedStreamRoute) Get(ctx context.Context, name string) (*v1.StreamRoute, error) { + return nil, fmt.Errorf("stream route operations not supported in virtual cluster") +} +func (u *unsupportedStreamRoute) List(ctx context.Context) ([]*v1.StreamRoute, error) { + return nil, fmt.Errorf("stream route operations not supported in virtual cluster") +} +func (u *unsupportedStreamRoute) Create(ctx context.Context, route *v1.StreamRoute) (*v1.StreamRoute, error) { + return nil, fmt.Errorf("stream route operations not supported in virtual cluster") +} +func (u *unsupportedStreamRoute) Delete(ctx context.Context, route *v1.StreamRoute) error { + return fmt.Errorf("stream route operations not supported in virtual cluster") +} +func (u *unsupportedStreamRoute) Update(ctx context.Context, route *v1.StreamRoute) (*v1.StreamRoute, error) { + return nil, fmt.Errorf("stream route operations not supported in virtual cluster") +} + +type unsupportedGlobalRule struct{} + +func (u *unsupportedGlobalRule) Get(ctx context.Context, id string) (*v1.GlobalRule, error) { + return nil, fmt.Errorf("global rule operations not supported in virtual cluster") +} +func (u *unsupportedGlobalRule) List(ctx context.Context) ([]*v1.GlobalRule, error) { + return nil, fmt.Errorf("global rule operations not supported in virtual cluster") +} +func (u *unsupportedGlobalRule) Create(ctx context.Context, rule *v1.GlobalRule) (*v1.GlobalRule, error) { + return nil, fmt.Errorf("global rule operations not supported in virtual cluster") +} +func (u *unsupportedGlobalRule) Delete(ctx context.Context, rule *v1.GlobalRule) error { + return fmt.Errorf("global rule operations not supported in virtual cluster") +} +func (u *unsupportedGlobalRule) Update(ctx context.Context, rule *v1.GlobalRule) (*v1.GlobalRule, error) { + return nil, fmt.Errorf("global rule operations not supported in virtual cluster") +} + +type unsupportedConsumer struct{} + +func (u *unsupportedConsumer) Get(ctx context.Context, name string) (*v1.Consumer, error) { + return nil, fmt.Errorf("consumer operations not supported in virtual cluster") +} +func (u *unsupportedConsumer) List(ctx context.Context) ([]*v1.Consumer, error) { + return nil, fmt.Errorf("consumer operations not supported in virtual cluster") +} +func (u *unsupportedConsumer) Create(ctx context.Context, consumer *v1.Consumer) (*v1.Consumer, error) { + return nil, fmt.Errorf("consumer operations not supported in virtual cluster") +} +func (u *unsupportedConsumer) Delete(ctx context.Context, consumer *v1.Consumer) error { + return fmt.Errorf("consumer operations not supported in virtual cluster") +} +func (u *unsupportedConsumer) Update(ctx context.Context, consumer *v1.Consumer) (*v1.Consumer, error) { + return nil, fmt.Errorf("consumer operations not supported in virtual cluster") +} + +type unsupportedPlugin struct{} + +func (u *unsupportedPlugin) List(ctx context.Context) ([]string, error) { + return nil, fmt.Errorf("plugin operations not supported in virtual cluster") +} + +type unsupportedPluginConfig struct{} + +func (u *unsupportedPluginConfig) Get(ctx context.Context, name string) (*v1.PluginConfig, error) { + return nil, fmt.Errorf("plugin config operations not supported in virtual cluster") +} +func (u *unsupportedPluginConfig) List(ctx context.Context) ([]*v1.PluginConfig, error) { + return nil, fmt.Errorf("plugin config operations not supported in virtual cluster") +} +func (u *unsupportedPluginConfig) Create(ctx context.Context, plugin *v1.PluginConfig) (*v1.PluginConfig, error) { + return nil, fmt.Errorf("plugin config operations not supported in virtual cluster") +} +func (u *unsupportedPluginConfig) Delete(ctx context.Context, plugin *v1.PluginConfig) error { + return fmt.Errorf("plugin config operations not supported in virtual cluster") +} +func (u *unsupportedPluginConfig) Update(ctx context.Context, plugin *v1.PluginConfig) (*v1.PluginConfig, error) { + return nil, fmt.Errorf("plugin config operations not supported in virtual cluster") +} + +type unsupportedSchema struct{} + +func (u *unsupportedSchema) GetPluginSchema(ctx context.Context, pluginName string) (*v1.Schema, error) { + return nil, fmt.Errorf("schema operations not supported in virtual cluster") +} +func (u *unsupportedSchema) GetRouteSchema(ctx context.Context) (*v1.Schema, error) { + return nil, fmt.Errorf("schema operations not supported in virtual cluster") +} +func (u *unsupportedSchema) GetUpstreamSchema(ctx context.Context) (*v1.Schema, error) { + return nil, fmt.Errorf("schema operations not supported in virtual cluster") +} +func (u *unsupportedSchema) GetConsumerSchema(ctx context.Context) (*v1.Schema, error) { + return nil, fmt.Errorf("schema operations not supported in virtual cluster") +} +func (u *unsupportedSchema) GetSslSchema(ctx context.Context) (*v1.Schema, error) { + return nil, fmt.Errorf("schema operations not supported in virtual cluster") +} +func (u *unsupportedSchema) GetPluginConfigSchema(ctx context.Context) (*v1.Schema, error) { + return nil, fmt.Errorf("schema operations not supported in virtual cluster") +} + +type unsupportedPluginMetadata struct{} + +func (u *unsupportedPluginMetadata) Get(ctx context.Context, name string) (*v1.PluginMetadata, error) { + return nil, fmt.Errorf("plugin metadata operations not supported in virtual cluster") +} +func (u *unsupportedPluginMetadata) List(ctx context.Context) ([]*v1.PluginMetadata, error) { + return nil, fmt.Errorf("plugin metadata operations not supported in virtual cluster") +} +func (u *unsupportedPluginMetadata) Create(ctx context.Context, metadata *v1.PluginMetadata) (*v1.PluginMetadata, error) { + return nil, fmt.Errorf("plugin metadata operations not supported in virtual cluster") +} +func (u *unsupportedPluginMetadata) Delete(ctx context.Context, metadata *v1.PluginMetadata) error { + return fmt.Errorf("plugin metadata operations not supported in virtual cluster") +} +func (u *unsupportedPluginMetadata) Update(ctx context.Context, metadata *v1.PluginMetadata) (*v1.PluginMetadata, error) { + return nil, fmt.Errorf("plugin metadata operations not supported in virtual cluster") +} + +type unsupportedValidator struct{} + +func (u *unsupportedValidator) ValidateStreamPluginSchema(plugins v1.Plugins) (bool, error) { + return false, fmt.Errorf("validation not supported in virtual cluster") +} +func (u *unsupportedValidator) ValidateHTTPPluginSchema(plugins v1.Plugins) (bool, error) { + return false, fmt.Errorf("validation not supported in virtual cluster") +} + +// APISIXDeployer implements Deployer interface for APISIX standalone +type APISIXDeployer struct { + t testing.TestingT + kubectlOpts *k8s.KubectlOptions + framework *framework.APISIXFramework + opts *DeployerOptions + + // APISIX-specific resources + service *corev1.Service + httpTunnel *k8s.Tunnel + httpsTunnel *k8s.Tunnel + adminClient *httpexpect.Expect +} + +// NewAPISIXDeployer creates a new APISIX deployer +func NewAPISIXDeployer(t testing.TestingT, kubectlOpts *k8s.KubectlOptions, framework *framework.APISIXFramework, opts *DeployerOptions) (Deployer, error) { + if opts == nil { + return nil, fmt.Errorf("deployer options cannot be nil") + } + + return &APISIXDeployer{ + t: t, + kubectlOpts: kubectlOpts, + framework: framework, + opts: opts, + }, nil +} + +// Deploy deploys APISIX in standalone mode +func (d *APISIXDeployer) Deploy(ctx context.Context) error { + // Deploy APISIX standalone using framework + frameworkDeployer := framework.NewAPISIXDeployer(d.t, d.kubectlOpts, &framework.APISIXDeployOptions{ + Namespace: d.opts.Namespace, + Image: d.opts.APISIXImage, + ServiceType: d.opts.ServiceType, + AdminKey: d.opts.AdminKey, + }) + + err := frameworkDeployer.Deploy(ctx) + if err != nil { + return fmt.Errorf("deploying APISIX: %w", err) + } + + // Get the service + d.service = frameworkDeployer.GetService() + + // Create tunnels + d.createTunnels() + + // Initialize admin client + d.initAdminClient() + + return nil +} + +// Cleanup cleans up APISIX resources +func (d *APISIXDeployer) Cleanup(ctx context.Context) error { + // Close tunnels + if d.httpTunnel != nil { + d.httpTunnel.Close() + } + if d.httpsTunnel != nil { + d.httpsTunnel.Close() + } + + // Delete APISIX deployment and service + // This will be handled by namespace deletion + return nil +} + +// GetHTTPClient returns HTTP client for APISIX gateway +func (d *APISIXDeployer) GetHTTPClient() *httpexpect.Expect { + if d.httpTunnel == nil { + d.createTunnels() + } + + u := url.URL{ + Scheme: "http", + Host: d.httpTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// GetHTTPSClient returns HTTPS client for APISIX gateway +func (d *APISIXDeployer) GetHTTPSClient(host string) *httpexpect.Expect { + if d.httpsTunnel == nil { + d.createTunnels() + } + + u := url.URL{ + Scheme: "https", + Host: d.httpsTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // accept any certificate; for testing only! + InsecureSkipVerify: true, + ServerName: host, + }, + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// GetAdminClient returns admin client for APISIX +func (d *APISIXDeployer) GetAdminClient() *httpexpect.Expect { + if d.adminClient == nil { + d.initAdminClient() + } + return d.adminClient +} + +// GetAdminKey returns admin key for APISIX +func (d *APISIXDeployer) GetAdminKey() string { + return d.opts.AdminKey +} + +// GetDataplaneResource returns dashboard cluster for admin operations +// For APISIX standalone, this creates a virtual cluster that directly talks to APISIX admin API +func (d *APISIXDeployer) GetDataplaneResource() dashboard.Cluster { + return &VirtualCluster{ + name: "default", + adminClient: d.GetAdminClient(), + adminKey: d.GetAdminKey(), + baseURL: fmt.Sprintf("http://%s", d.getAdminEndpoint()), + } +} + +// GetDataplaneResourceHTTPS returns HTTPS dashboard cluster +func (d *APISIXDeployer) GetDataplaneResourceHTTPS() dashboard.Cluster { + return &VirtualCluster{ + name: "default-https", + adminClient: d.GetAdminClient(), // For now, use the same admin client + adminKey: d.GetAdminKey(), + baseURL: fmt.Sprintf("https://%s", d.getAdminEndpoint()), + httpsMode: true, + } +} + +// GetMode returns the deployment mode +func (d *APISIXDeployer) GetMode() DeployMode { + return DeployModeAPISIX +} + +// createTunnels creates HTTP and HTTPS tunnels for APISIX +func (d *APISIXDeployer) createTunnels() { + if d.service == nil { + return + } + + // Create HTTP tunnel + httpTunnel := k8s.NewTunnel(d.kubectlOpts, k8s.ResourceTypeService, d.service.Name, 9080, 9080) + httpTunnel.ForwardPort(d.t) + d.httpTunnel = httpTunnel + + // Create HTTPS tunnel + httpsTunnel := k8s.NewTunnel(d.kubectlOpts, k8s.ResourceTypeService, d.service.Name, 9443, 9443) + httpsTunnel.ForwardPort(d.t) + d.httpsTunnel = httpsTunnel +} + +// initAdminClient initializes the admin client +func (d *APISIXDeployer) initAdminClient() { + if d.service == nil { + return + } + + // Create admin tunnel + adminTunnel := k8s.NewTunnel(d.kubectlOpts, k8s.ResourceTypeService, d.service.Name, 9180, 9180) + adminTunnel.ForwardPort(d.t) + + u := url.URL{ + Scheme: "http", + Host: adminTunnel.Endpoint(), + } + d.adminClient = httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// getAdminEndpoint returns the admin endpoint for APISIX +func (d *APISIXDeployer) getAdminEndpoint() string { + // Create admin tunnel if not exists + if d.service != nil { + adminTunnel := k8s.NewTunnel(d.kubectlOpts, k8s.ResourceTypeService, d.service.Name, 0, 9180) + adminTunnel.ForwardPort(d.t) + return adminTunnel.Endpoint() + } + return "localhost:9180" +} diff --git a/test/e2e/scaffold/apisix_scafflod.go b/test/e2e/scaffold/apisix_scafflod.go new file mode 100644 index 000000000..19dce48af --- /dev/null +++ b/test/e2e/scaffold/apisix_scafflod.go @@ -0,0 +1,520 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "context" + "crypto/tls" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/gavv/httpexpect/v2" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/testing" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apache/apisix-ingress-controller/pkg/dashboard" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" +) + +// APISIXScaffold implements TestScaffold for APISIX standalone +type APISIXScaffold struct { + *framework.APISIXFramework + + opts *Options + kubectlOptions *k8s.KubectlOptions + namespace string + t testing.TestingT + nodes []corev1.Node + + finalizers []func() + + // Use the new Deployer interface + deployer Deployer + adminClient *httpexpect.Expect + apisixHttpTunnel *k8s.Tunnel + apisixHttpsTunnel *k8s.Tunnel + httpbinService *corev1.Service +} + +// NewAPISIXScaffold creates a new APISIX scaffold +func NewAPISIXScaffold(opts *Options) *APISIXScaffold { + if opts == nil { + opts = &Options{} + } + + // Set default values + if opts.Name == "" { + opts.Name = "default" + } + if opts.IngressAPISIXReplicas <= 0 { + opts.IngressAPISIXReplicas = 1 + } + if opts.HTTPBinServicePort == 0 { + opts.HTTPBinServicePort = 80 + } + + s := &APISIXScaffold{ + APISIXFramework: framework.GetAPISIXFramework(), + opts: opts, + t: GinkgoT(), + } + + BeforeEach(s.BeforeEach) + AfterEach(s.AfterEach) + + return s +} + +// BeforeEach sets up the test environment for each test +func (s *APISIXScaffold) BeforeEach() { + var err error + + s.namespace = fmt.Sprintf("apisix-e2e-tests-%s-%d", s.opts.Name, time.Now().Nanosecond()) + s.kubectlOptions = &k8s.KubectlOptions{ + ConfigPath: s.opts.Kubeconfig, + Namespace: s.namespace, + } + + s.finalizers = nil + + // Create test namespace + k8s.CreateNamespace(s.t, s.kubectlOptions, s.namespace) + + s.nodes, err = k8s.GetReadyNodesE(s.t, s.kubectlOptions) + Expect(err).NotTo(HaveOccurred(), "getting ready nodes") + + // Deploy APISIX standalone + s.deployAPISIXStandalone() + + // Deploy ingress controller + s.DeployIngress(framework.IngressDeployOpts{ + ControllerName: s.getControllerName(), + Namespace: s.namespace, + Replicas: s.opts.IngressAPISIXReplicas, + }) + + // Deploy test services + s.deployTestServices() +} + +// AfterEach cleans up after each test +func (s *APISIXScaffold) AfterEach() { + defer GinkgoRecover() + + if CurrentSpecReport().Failed() { + s.Logf("Test failed, dumping logs") + output := s.getDeploymentLogs("apisix-ingress-controller") + if output != "" { + _, _ = fmt.Fprintln(GinkgoWriter, output) + } + } + + // Clean up namespace + err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) + Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) + + // Run finalizers + for i := len(s.finalizers) - 1; i >= 0; i-- { + s.runWithRecover(s.finalizers[i]) + } + + // Wait to prevent overwhelming the worker node + time.Sleep(3 * time.Second) +} + +// NewAPISIXClient returns HTTP client for APISIX +func (s *APISIXScaffold) NewAPISIXClient() *httpexpect.Expect { + u := url.URL{ + Scheme: "http", + Host: s.apisixHttpTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// NewAPISIXHttpsClient returns HTTPS client for APISIX +func (s *APISIXScaffold) NewAPISIXHttpsClient(host string) *httpexpect.Expect { + u := url.URL{ + Scheme: "https", + Host: s.apisixHttpsTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // accept any certificate; for testing only! + InsecureSkipVerify: true, + ServerName: host, + }, + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }) +} + +// DeployIngress deploys the ingress controller +func (s *APISIXScaffold) DeployIngress(opts framework.IngressDeployOpts) { + s.APISIXFramework.DeployIngress(opts) +} + +// deployAPISIXStandalone deploys APISIX in standalone mode +func (s *APISIXScaffold) deployAPISIXStandalone() { + // Create our new APISIX deployer + deployer, err := NewAPISIXDeployer(s.t, s.kubectlOptions, s.APISIXFramework, &DeployerOptions{ + Namespace: s.namespace, + ServiceType: "ClusterIP", + AdminKey: s.opts.APISIXAdminAPIKey, + APISIXImage: "apache/apisix:3.8.0", // Default image + }) + Expect(err).NotTo(HaveOccurred(), "creating APISIX deployer") + + s.deployer = deployer + + err = s.deployer.Deploy(context.Background()) + Expect(err).NotTo(HaveOccurred(), "deploying APISIX standalone") + + // Create HTTP tunnels for APISIX - we need to get the service from the deployer + apisixDeployer, ok := s.deployer.(*APISIXDeployer) + if !ok { + panic("deployer is not APISIXDeployer") + } + s.createAPISIXTunnel(apisixDeployer.service) + + // init admin client + s.initAdminClient() +} + +// createAPISIXTunnel creates HTTP tunnel to APISIX service +func (s *APISIXScaffold) createAPISIXTunnel(svc *corev1.Service) { + var ( + httpNodePort int + httpsNodePort int + httpPort int + httpsPort int + ) + + for _, port := range svc.Spec.Ports { + switch port.Name { + case "http": + httpNodePort = int(port.NodePort) + httpPort = int(port.Port) + case "https": + httpsNodePort = int(port.NodePort) + httpsPort = int(port.Port) + } + } + + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, svc.Name, httpNodePort, httpPort) + tunnel.ForwardPort(s.t) + s.apisixHttpTunnel = tunnel + + httpsTunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, svc.Name, httpsNodePort, httpsPort) + httpsTunnel.ForwardPort(s.t) + s.apisixHttpsTunnel = httpsTunnel + + s.addFinalizers(func() { + tunnel.Close() + httpsTunnel.Close() + }) +} + +// initAdminClient initializes the APISIX admin client +func (s *APISIXScaffold) initAdminClient() { + adminTunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix", 0, 9180) + adminTunnel.ForwardPort(s.t) + + s.addFinalizers(func() { + adminTunnel.Close() + }) + + adminEndpoint := fmt.Sprintf("http://%s", adminTunnel.Endpoint()) + s.adminClient = httpexpect.Default(GinkgoT(), adminEndpoint) +} + +// deployTestServices deploys test services like httpbin +func (s *APISIXScaffold) deployTestServices() { + s.Logf("Deploying test services") + + // Deploy httpbin service for testing + httpbinYaml := ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + template: + metadata: + labels: + app: httpbin + spec: + containers: + - name: httpbin + image: kennethreitz/httpbin:latest + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: httpbin +spec: + selector: + app: httpbin + ports: + - port: 80 + targetPort: 80 + type: ClusterIP +` + + err := s.CreateResourceFromString(httpbinYaml) + if err != nil { + s.Logf("Failed to deploy httpbin: %v", err) + } else { + s.Logf("httpbin service deployed successfully") + } +} + +// getControllerName returns the controller name +func (s *APISIXScaffold) getControllerName() string { + if s.opts.ControllerName == "" { + return fmt.Sprintf("%s/%d", DefaultControllerName, time.Now().Nanosecond()) + } + return s.opts.ControllerName +} + +// addFinalizers adds cleanup functions +func (s *APISIXScaffold) addFinalizers(f func()) { + s.finalizers = append(s.finalizers, f) +} + +// runWithRecover runs a function with panic recovery +func (s *APISIXScaffold) runWithRecover(f func()) { + defer func() { + if r := recover(); r != nil { + s.Logf("Recovered from panic in finalizer: %v", r) + } + }() + f() +} + +// getDeploymentLogs gets logs from a deployment +func (s *APISIXScaffold) getDeploymentLogs(name string) string { + cli, err := k8s.GetKubernetesClientE(s.t) + if err != nil { + return "" + } + + pods, err := cli.CoreV1().Pods(s.namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app=" + name, + }) + if err != nil { + return "" + } + + var logs string + for _, pod := range pods.Items { + logs += fmt.Sprintf("=== pod: %s ===\n", pod.Name) + podLogs, err := cli.CoreV1().RESTClient().Get(). + Resource("pods"). + Namespace(s.namespace). + Name(pod.Name).SubResource("log"). + Do(context.TODO()). + Raw() + if err == nil { + logs += string(podLogs) + } + logs += "\n" + } + return logs +} + +func (s *APISIXScaffold) AdminKey() string { + return s.opts.APISIXAdminAPIKey +} + +func (s *APISIXScaffold) GetControllerName() string { + return s.getControllerName() +} + +func (s *APISIXScaffold) Namespace() string { + return s.namespace +} + +func (s *APISIXScaffold) GetContext() context.Context { + return s.Context +} + +func (s *APISIXScaffold) CreateResourceFromString(resourceYaml string) error { + return s.CreateResourceFromStringWithNamespace(resourceYaml, s.namespace) +} + +func (s *APISIXScaffold) CreateResourceFromStringWithNamespace(resourceYaml, namespace string) error { + kubectlOpts := *s.kubectlOptions + if namespace != "" { + kubectlOpts.Namespace = namespace + } + return k8s.KubectlApplyFromStringE(s.t, &kubectlOpts, resourceYaml) +} + +func (s *APISIXScaffold) DeleteResourceFromString(resourceYaml string) error { + return s.DeleteResourceFromStringWithNamespace(resourceYaml, s.namespace) +} + +func (s *APISIXScaffold) DeleteResourceFromStringWithNamespace(resourceYaml, namespace string) error { + kubectlOpts := *s.kubectlOptions + if namespace != "" { + kubectlOpts.Namespace = namespace + } + return k8s.KubectlDeleteFromStringE(s.t, &kubectlOpts, resourceYaml) +} + +func (s *APISIXScaffold) DeleteResource(resourceType, name string) error { + args := []string{"delete", resourceType, name} + return k8s.RunKubectlE(s.t, s.kubectlOptions, args...) +} + +func (s *APISIXScaffold) GetResourceYaml(resourceType, name string) (string, error) { + return s.GetResourceYamlFromNamespace(resourceType, name, s.namespace) +} + +func (s *APISIXScaffold) GetResourceYamlFromNamespace(resourceType, name, namespace string) (string, error) { + kubectlOpts := *s.kubectlOptions + if namespace != "" { + kubectlOpts.Namespace = namespace + } + args := []string{"get", resourceType, name, "-o", "yaml"} + return k8s.RunKubectlAndGetOutputE(s.t, &kubectlOpts, args...) +} + +func (s *APISIXScaffold) RunKubectlAndGetOutput(args ...string) (string, error) { + return k8s.RunKubectlAndGetOutputE(s.t, s.kubectlOptions, args...) +} + +func (s *APISIXScaffold) NewKubeTlsSecret(secretName, cert, key string) error { + const kubeTlsSecretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: %s +type: kubernetes.io/tls +data: + tls.crt: %s + tls.key: %s +` + certBase64 := base64.StdEncoding.EncodeToString([]byte(cert)) + keyBase64 := base64.StdEncoding.EncodeToString([]byte(key)) + secret := fmt.Sprintf(kubeTlsSecretTemplate, secretName, certBase64, keyBase64) + return s.CreateResourceFromString(secret) +} + +func (s *APISIXScaffold) DefaultDataplaneResource() dashboard.Cluster { + if s.deployer != nil { + return s.deployer.GetDataplaneResource() + } + return nil +} + +func (s *APISIXScaffold) CreateAdditionalGatewayGroup(namePrefix string) (string, string, error) { + // APISIX standalone doesn't support additional gateway groups + // This method is API7-specific functionality + return "", "", fmt.Errorf("additional gateway groups not supported in APISIX standalone mode") +} + +func (s *APISIXScaffold) GetAdditionalGatewayGroup(gatewayGroupID string) (*GatewayGroupResources, bool) { + // APISIX standalone doesn't support additional gateway groups + return nil, false +} + +func (s *APISIXScaffold) NewAPISIXClientForGatewayGroup(gatewayGroupID string) (*httpexpect.Expect, error) { + // APISIX standalone doesn't support additional gateway groups + // Return the main APISIX client instead + return s.NewAPISIXClient(), nil +} + +func (s *APISIXScaffold) GetGinkgoT() ginkgo.GinkgoTInterface { + return ginkgo.GinkgoT() +} + +func (s *APISIXScaffold) GetK8sClient() client.Client { + return s.APISIXFramework.K8sClient +} + +func (s *APISIXScaffold) ApplyDefaultGatewayResource(gatewayProxy, gatewayClass, gateway, httpRoute string) { + // For APISIX standalone, we don't need to apply gateway resources + // since it doesn't use the gateway API like API7 + s.Logf("ApplyDefaultGatewayResource called but not implemented for APISIX standalone") +} + +func (s *APISIXScaffold) DefaultDataplaneResourceHTTPS() dashboard.Cluster { + if s.deployer != nil { + return s.deployer.GetDataplaneResourceHTTPS() + } + return nil +} + +func (s *APISIXScaffold) ResourceApplied(resourType, resourceName, resourceRaw string, observedGeneration int) { + // Simple implementation for APISIX standalone + // This can be enhanced if needed for specific tests + s.Logf("Resource applied: %s/%s", resourType, resourceName) +} + +func (s *APISIXScaffold) ScaleIngress(replicas int) { + // Scale the ingress controller deployment + args := []string{"scale", "deployment", "apisix-ingress-controller", "--replicas", fmt.Sprintf("%d", replicas)} + err := k8s.RunKubectlE(s.t, s.kubectlOptions, args...) + if err != nil { + s.Logf("Failed to scale ingress controller: %v", err) + } +} + +func (s *APISIXScaffold) GetDeploymentLogs(name string) string { + return s.getDeploymentLogs(name) +} + +func (s *APISIXScaffold) DeployNginx(options framework.NginxOptions) { + // Deploy nginx test service for APISIX standalone + // Since APISIXFramework doesn't have DeployNginx, we need to implement it manually + // For now, just log that it's called + s.Logf("DeployNginx called with options: %+v - not implemented for APISIX standalone", options) +} + +// GetDeployer returns the underlying deployer instance +func (s *APISIXScaffold) GetDeployer() Deployer { + return s.deployer +} diff --git a/test/e2e/scaffold/common.go b/test/e2e/scaffold/common.go new file mode 100644 index 000000000..004702cf6 --- /dev/null +++ b/test/e2e/scaffold/common.go @@ -0,0 +1,165 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "context" + "encoding/base64" + "fmt" + + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/testing" + "github.com/onsi/ginkgo/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + kubeTlsSecretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: %s +type: kubernetes.io/tls +data: + tls.crt: %s + tls.key: %s +` +) + +// BaseScaffold provides common implementation for scaffold methods +type BaseScaffold struct { + t testing.TestingT + kubectlOpts *k8s.KubectlOptions + namespace string + opts *Options +} + +// Common resource management methods that can be shared between scaffolds + +// CreateResourceFromString creates a K8s resource from YAML string +func (b *BaseScaffold) CreateResourceFromString(resourceYaml string) error { + return b.CreateResourceFromStringWithNamespace(resourceYaml, b.namespace) +} + +// CreateResourceFromStringWithNamespace creates a K8s resource from YAML string in specified namespace +func (b *BaseScaffold) CreateResourceFromStringWithNamespace(resourceYaml, namespace string) error { + kubectlOpts := *b.kubectlOpts + if namespace != "" { + kubectlOpts.Namespace = namespace + } + return k8s.KubectlApplyFromStringE(b.t, &kubectlOpts, resourceYaml) +} + +// DeleteResourceFromString deletes a K8s resource from YAML string +func (b *BaseScaffold) DeleteResourceFromString(resourceYaml string) error { + return b.DeleteResourceFromStringWithNamespace(resourceYaml, b.namespace) +} + +// DeleteResourceFromStringWithNamespace deletes a K8s resource from YAML string in specified namespace +func (b *BaseScaffold) DeleteResourceFromStringWithNamespace(resourceYaml, namespace string) error { + kubectlOpts := *b.kubectlOpts + if namespace != "" { + kubectlOpts.Namespace = namespace + } + return k8s.KubectlDeleteFromStringE(b.t, &kubectlOpts, resourceYaml) +} + +// DeleteResource deletes a K8s resource by type and name +func (b *BaseScaffold) DeleteResource(resourceType, name string) error { + args := []string{"delete", resourceType, name} + return k8s.RunKubectlE(b.t, b.kubectlOpts, args...) +} + +// GetResourceYaml gets a K8s resource YAML by type and name +func (b *BaseScaffold) GetResourceYaml(resourceType, name string) (string, error) { + return b.GetResourceYamlFromNamespace(resourceType, name, b.namespace) +} + +// GetResourceYamlFromNamespace gets a K8s resource YAML by type, name and namespace +func (b *BaseScaffold) GetResourceYamlFromNamespace(resourceType, name, namespace string) (string, error) { + kubectlOpts := *b.kubectlOpts + if namespace != "" { + kubectlOpts.Namespace = namespace + } + args := []string{"get", resourceType, name, "-o", "yaml"} + return k8s.RunKubectlAndGetOutputE(b.t, &kubectlOpts, args...) +} + +// RunKubectlAndGetOutput runs kubectl command and returns output +func (b *BaseScaffold) RunKubectlAndGetOutput(args ...string) (string, error) { + return k8s.RunKubectlAndGetOutputE(b.t, b.kubectlOpts, args...) +} + +// NewKubeTlsSecret creates a TLS secret +func (b *BaseScaffold) NewKubeTlsSecret(secretName, cert, key string) error { + certBase64 := base64.StdEncoding.EncodeToString([]byte(cert)) + keyBase64 := base64.StdEncoding.EncodeToString([]byte(key)) + secret := fmt.Sprintf(kubeTlsSecretTemplate, secretName, certBase64, keyBase64) + return b.CreateResourceFromString(secret) +} + +// Namespace returns the current namespace +func (b *BaseScaffold) Namespace() string { + return b.namespace +} + +// GetContext returns the context +func (b *BaseScaffold) GetContext() context.Context { + return context.TODO() +} + +// GetGinkgoT returns the Ginkgo test interface +func (b *BaseScaffold) GetGinkgoT() ginkgo.GinkgoTInterface { + return ginkgo.GinkgoT() +} + +// GetK8sClient returns the Kubernetes client +func (b *BaseScaffold) GetK8sClient() client.Client { + // This needs to be implemented by the concrete scaffold implementations + // since they have access to the framework with the client + panic("GetK8sClient not implemented in BaseScaffold - should be implemented by concrete scaffolds") +} + +// GetControllerName returns the controller name +func (b *BaseScaffold) GetControllerName() string { + if b.opts != nil && b.opts.ControllerName != "" { + return b.opts.ControllerName + } + return DefaultControllerName +} + +// AdminKey returns the admin key +func (b *BaseScaffold) AdminKey() string { + if b.opts != nil && b.opts.APISIXAdminAPIKey != "" { + return b.opts.APISIXAdminAPIKey + } + return "edd1c9f034335f136f87ad84b625c8f1" // Default APISIX admin key +} + +// TODO: These methods need specific implementations in each scaffold +// and should not be in the base scaffold as they depend on the deployment mode + +// ResourceApplied is a placeholder - should be implemented in specific scaffolds +func (b *BaseScaffold) ResourceApplied(resourType, resourceName, resourceRaw string, observedGeneration int) { + // This method needs specific implementation based on deployment mode +} + +// ApplyDefaultGatewayResource is a placeholder - should be implemented in specific scaffolds +func (b *BaseScaffold) ApplyDefaultGatewayResource(gatewayProxy, gatewayClass, gateway, httpRoute string) { + // This method needs specific implementation based on deployment mode +} + +// ScaleIngress is a placeholder - should be implemented in specific scaffolds +func (b *BaseScaffold) ScaleIngress(replicas int) { + // This method needs specific implementation based on deployment mode +} diff --git a/test/e2e/scaffold/deployer.go b/test/e2e/scaffold/deployer.go new file mode 100644 index 000000000..c3547afe5 --- /dev/null +++ b/test/e2e/scaffold/deployer.go @@ -0,0 +1,134 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "context" + "fmt" + "os" + + "github.com/gavv/httpexpect/v2" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/testing" + + "github.com/apache/apisix-ingress-controller/pkg/dashboard" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" +) + +// DeployMode defines the deployment mode +type DeployMode string + +const ( + DeployModeAPI7 DeployMode = "api7" + DeployModeAPISIX DeployMode = "apisix" +) + +// Deployer defines the interface for deploying data plane components +type Deployer interface { + // Deploy deploys the data plane components + // for api7 mode: deploy api7 dashboard and api7 gateway + // for apisix mode: deploy apisix dp only + Deploy(ctx context.Context) error + + // Cleanup cleans up deployed resources + Cleanup(ctx context.Context) error + + // GetHTTPClient returns HTTP client for the gateway data plane + GetHTTPClient() *httpexpect.Expect + + // GetHTTPSClient returns HTTPS client for the gateway data plane + GetHTTPSClient(host string) *httpexpect.Expect + + // GetAdminClient returns admin client for configuration + GetAdminClient() *httpexpect.Expect + + // GetAdminKey returns admin key for authentication + GetAdminKey() string + + // GetDataplaneResource returns dashboard cluster for admin operations + GetDataplaneResource() dashboard.Cluster + GetDataplaneResourceHTTPS() dashboard.Cluster + + // GetMode returns the deployment mode + GetMode() DeployMode +} + +// DeployerFactory creates deployers based on mode +type DeployerFactory interface { + CreateDeployer(mode DeployMode, opts *DeployerOptions) (Deployer, error) +} + +// DeployerOptions contains options for deployer creation +type DeployerOptions struct { + Namespace string + AdminKey string + ControllerName string + // API7-specific options + GatewayGroupID string + DashboardAddr string + // APISIX-specific options + APISIXImage string + ServiceType string +} + +// defaultDeployerFactory is the default factory implementation +type defaultDeployerFactory struct { + t testing.TestingT + kubectlOpts *k8s.KubectlOptions + framework interface{} +} + +// NewDeployerFactory creates a new deployer factory +func NewDeployerFactory(t testing.TestingT, kubectlOpts *k8s.KubectlOptions, framework interface{}) DeployerFactory { + return &defaultDeployerFactory{ + t: t, + kubectlOpts: kubectlOpts, + framework: framework, + } +} + +// CreateDeployer creates a deployer based on the specified mode +func (f *defaultDeployerFactory) CreateDeployer(mode DeployMode, opts *DeployerOptions) (Deployer, error) { + switch mode { + case DeployModeAPI7: + // Extract API7 framework + api7Framework, ok := f.framework.(*framework.Framework) + if !ok { + return nil, fmt.Errorf("invalid framework type for API7 mode") + } + return NewAPI7Deployer(f.t, f.kubectlOpts, api7Framework, opts) + case DeployModeAPISIX: + // Extract APISIX framework + apisixFramework, ok := f.framework.(*framework.APISIXFramework) + if !ok { + return nil, fmt.Errorf("invalid framework type for APISIX mode") + } + return NewAPISIXDeployer(f.t, f.kubectlOpts, apisixFramework, opts) + default: + return nil, fmt.Errorf("unsupported deploy mode: %s", mode) + } +} + +// GetDeployModeFromEnv returns deployment mode from environment variable +func GetDeployModeFromEnv() DeployMode { + mode := os.Getenv("DEPLOY_MODE") + switch mode { + case "api7": + return DeployModeAPI7 + case "apisix": + return DeployModeAPISIX + default: + // Default to API7 for backward compatibility + return DeployModeAPI7 + } +} diff --git a/test/e2e/scaffold/interface.go b/test/e2e/scaffold/interface.go new file mode 100644 index 000000000..e6f44bc91 --- /dev/null +++ b/test/e2e/scaffold/interface.go @@ -0,0 +1,66 @@ +package scaffold + +import ( + "context" + + "github.com/gavv/httpexpect/v2" + "github.com/onsi/ginkgo/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apache/apisix-ingress-controller/pkg/dashboard" + "github.com/apache/apisix-ingress-controller/test/e2e/framework" +) + +var NewScaffold func(*Options) TestScaffold + +// TestScaffold defines the interface for test scaffold implementations +type TestScaffold interface { + // HTTP client methods - common interface but implementation may differ + NewAPISIXClient() *httpexpect.Expect + NewAPISIXHttpsClient(host string) *httpexpect.Expect + + // Basic operation methods - common to all implementations + AdminKey() string + GetControllerName() string + Namespace() string + GetContext() context.Context + GetGinkgoT() ginkgo.GinkgoTInterface + GetK8sClient() client.Client + + // Resource management methods - common to all implementations + CreateResourceFromString(resourceYaml string) error + CreateResourceFromStringWithNamespace(resourceYaml, namespace string) error + DeleteResourceFromString(resourceYaml string) error + DeleteResourceFromStringWithNamespace(resourceYaml, namespace string) error + DeleteResource(resourceType, name string) error + GetResourceYaml(resourceType, name string) (string, error) + GetResourceYamlFromNamespace(resourceType, name, namespace string) (string, error) + ResourceApplied(resourType, resourceName, resourceRaw string, observedGeneration int) + ApplyDefaultGatewayResource(gatewayProxy, gatewayClass, gateway, httpRoute string) + GetDeploymentLogs(name string) string + + // Kubernetes operation methods - common to all implementations + RunKubectlAndGetOutput(args ...string) (string, error) + NewKubeTlsSecret(secretName, cert, key string) error + + // Dataplane resource access methods - common interface + DefaultDataplaneResource() dashboard.Cluster + DefaultDataplaneResourceHTTPS() dashboard.Cluster + + // Common infrastructure methods + ScaleIngress(int) + DeployNginx(options framework.NginxOptions) + + // Access to underlying deployer + GetDeployer() Deployer + + // TODO: These methods should be deprecated for multi-gateway support + // They are API7-specific and should be moved to a separate interface + CreateAdditionalGatewayGroup(namePrefix string) (string, string, error) + GetAdditionalGatewayGroup(gatewayGroupID string) (*GatewayGroupResources, bool) + NewAPISIXClientForGatewayGroup(gatewayGroupID string) (*httpexpect.Expect, error) +} + +//type TestFrameWork interface { +// NewScaffold(*Options) TestScaffold +//}