diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index e4e8aa522..e4cfe22ae 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -70,6 +70,18 @@ jobs: echo "building images..." make build-image + - name: Build Dockerfile.dev and extract adc binary + run: | + echo "Building Dockerfile.dev..." + docker build -f Dockerfile.dev -t adc-builder:latest . + echo "Extracting adc binary..." + docker run --name adc-temp --entrypoint="" adc-builder:latest /bin/true + docker cp adc-temp:/bin/adc ./bin/adc + docker rm adc-temp + chmod +x ./bin/adc + mv ./bin/adc /usr/local/bin/adc + echo "ADC binary extracted to /usr/local/bin/adc" + - name: Launch Kind Cluster run: | make kind-up diff --git a/Dockerfile.dev b/Dockerfile.dev index b7f1e15c1..ca2757643 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -3,7 +3,7 @@ ARG ENABLE_PROXY=false FROM node:22 AS node_builder ARG TARGETARCH -ARG ADC_COMMIT=78484e87a0168e0f86d130bfae8f808d0d1a1e41 +ARG ADC_COMMIT=e948079ed0576dbac29320ebfa01c9b7a298924c WORKDIR /app @@ -22,10 +22,13 @@ RUN apt update \ && rm -rf /app FROM debian:bullseye-slim + +ARG TARGETARCH + WORKDIR /app COPY --from=node_builder /bin/adc /bin/adc -COPY ./bin/apisix-ingress-controller . +COPY ./bin/apisix-ingress-controller_${TARGETARCH} ./apisix-ingress-controller COPY ./config/samples/config.yaml ./conf/config.yaml ENTRYPOINT ["/app/apisix-ingress-controller"] diff --git a/test/e2e/framework/manifests/apisix-standalone.yaml b/test/e2e/framework/manifests/apisix-standalone.yaml index 299c619e0..912aa8325 100644 --- a/test/e2e/framework/manifests/apisix-standalone.yaml +++ b/test/e2e/framework/manifests/apisix-standalone.yaml @@ -4,21 +4,20 @@ metadata: name: apisix-conf data: config.yaml: | - apisix: - enable_admin: true - admin_key: - - name: admin - key: {{ .AdminKey }} - role: admin - ssl: - enabled: true - nginx_config: - worker_processes: 2 - error_log_level: info deployment: role: traditional role_traditional: config_provider: yaml + admin: + allow_admin: + - 0.0.0.0/0 + admin_key: + - key: {{ .AdminKey }} + name: admin + role: admin + nginx_config: + worker_processes: 2 + error_log_level: info --- apiVersion: apps/v1 kind: Deployment diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index dc3f7b46b..353bd62de 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -204,7 +204,8 @@ spec: tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) assert.Nil(GinkgoT(), err, "list tls error") assert.Len(GinkgoT(), tls, 1, "tls number not expect") - assert.Equal(GinkgoT(), Cert, tls[0].Cert, "tls cert not expect") + assert.Len(GinkgoT(), tls[0].Certificates, 1, "length of certificates not expect") + assert.Equal(GinkgoT(), Cert, tls[0].Certificates[0].Certificate, "tls cert not expect") assert.ElementsMatch(GinkgoT(), []string{host, "*.api6.com"}, tls[0].Snis) }) @@ -277,7 +278,8 @@ spec: tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) assert.Nil(GinkgoT(), err, "list tls error") assert.Len(GinkgoT(), tls, 1, "tls number not expect") - assert.Equal(GinkgoT(), Cert, tls[0].Cert, "tls cert not expect") + assert.Len(GinkgoT(), tls[0].Certificates, 1, "length of certificates not expect") + assert.Equal(GinkgoT(), Cert, tls[0].Certificates[0].Certificate, "tls cert not expect") assert.Equal(GinkgoT(), tls[0].Labels["k8s/controller-name"], "apisix.apache.org/apisix-ingress-controller") By("update secret") @@ -289,7 +291,10 @@ spec: if len(tls) < 1 { return "" } - return tls[0].Cert + if len(tls[0].Certificates) < 1 { + return "" + } + return tls[0].Certificates[0].Certificate }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(framework.TestCert)) }) }) diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index a53d3fa26..93b0f6237 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -131,7 +131,8 @@ spec: tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) assert.Nil(GinkgoT(), err, "list tls error") assert.Len(GinkgoT(), tls, 1, "tls number not expect") - assert.Equal(GinkgoT(), Cert, tls[0].Cert, "tls cert not expect") + assert.Len(GinkgoT(), tls[0].Certificates, 1, "length of certificates not expect") + assert.Equal(GinkgoT(), Cert, tls[0].Certificates[0].Certificate, "tls cert not expect") assert.ElementsMatch(GinkgoT(), []string{host}, tls[0].Snis) }) }) diff --git a/test/e2e/scaffold/adc.go b/test/e2e/scaffold/adc.go new file mode 100644 index 000000000..aab8b50a1 --- /dev/null +++ b/test/e2e/scaffold/adc.go @@ -0,0 +1,236 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "time" + + "github.com/api7/gopkg/pkg/log" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" + "github.com/apache/apisix-ingress-controller/internal/provider/adc/translator" +) + +// DataplaneResource defines the interface for accessing dataplane resources +type DataplaneResource interface { + Route() RouteResource + Service() ServiceResource + SSL() SSLResource + Consumer() ConsumerResource +} + +// RouteResource defines the interface for route resources +type RouteResource interface { + List(ctx context.Context) ([]*adctypes.Route, error) +} + +// ServiceResource defines the interface for service resources +type ServiceResource interface { + List(ctx context.Context) ([]*adctypes.Service, error) +} + +// SSLResource defines the interface for SSL resources +type SSLResource interface { + List(ctx context.Context) ([]*adctypes.SSL, error) +} + +// ConsumerResource defines the interface for consumer resources +type ConsumerResource interface { + List(ctx context.Context) ([]*adctypes.Consumer, error) +} + +// adcDataplaneResource implements DataplaneResource using ADC command +type adcDataplaneResource struct { + backend string + serverAddr string + token string + tlsVerify bool + syncTimeout time.Duration +} + +// newADCDataplaneResource creates a new ADC-based dataplane resource +func newADCDataplaneResource(backend, serverAddr, token string, tlsVerify bool) DataplaneResource { + return &adcDataplaneResource{ + backend: backend, + serverAddr: serverAddr, + token: token, + tlsVerify: tlsVerify, + syncTimeout: 30 * time.Second, + } +} + +func (a *adcDataplaneResource) Route() RouteResource { + return &adcRouteResource{a} +} + +func (a *adcDataplaneResource) Service() ServiceResource { + return &adcServiceResource{a} +} + +func (a *adcDataplaneResource) SSL() SSLResource { + return &adcSSLResource{a} +} + +func (a *adcDataplaneResource) Consumer() ConsumerResource { + return &adcConsumerResource{a} +} + +func init() { + // dashboard sdk log + l, err := log.NewLogger( + log.WithOutputFile("stderr"), + log.WithLogLevel("debug"), + log.WithSkipFrames(3), + ) + if err != nil { + panic(err) + } + log.DefaultLogger = l +} + +// dumpResources executes adc dump command and returns the resources +func (a *adcDataplaneResource) dumpResources(ctx context.Context) (*translator.TranslateResult, error) { + ctxWithTimeout, cancel := context.WithTimeout(ctx, a.syncTimeout) + defer cancel() + + // Create a temporary file for the adc dump + tempFile, err := os.CreateTemp("", "adc-dump-*.yaml") + if err != nil { + return nil, err + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() + + args := []string{"dump", "-o", tempFile.Name()} + if !a.tlsVerify { + args = append(args, "--tls-skip-verify") + } + + adcEnv := []string{ + "ADC_RUNNING_MODE=", // need to set empty + "ADC_BACKEND=" + a.backend, + "ADC_SERVER=" + a.serverAddr, + "ADC_TOKEN=" + a.token, + } + + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctxWithTimeout, "adc", args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, adcEnv...) + + log.Debug("running adc command", zap.String("command", cmd.String()), zap.Strings("env", adcEnv)) + + if err := cmd.Run(); err != nil { + stderrStr := stderr.String() + stdoutStr := stdout.String() + errMsg := stderrStr + if errMsg == "" { + errMsg = stdoutStr + } + log.Errorw("failed to run adc", + zap.Error(err), + zap.String("output", stdoutStr), + zap.String("stderr", stderrStr), + ) + return nil, errors.New("failed to dump resources: " + errMsg + ", exit err: " + err.Error()) + } + + // Read the YAML file that was created by adc dump + yamlData, err := os.ReadFile(tempFile.Name()) + if err != nil { + return nil, err + } + + var resources adctypes.Resources + if err := yaml.Unmarshal(yamlData, &resources); err != nil { + return nil, err + } + + // Extract routes from services + var routes []*adctypes.Route + for _, service := range resources.Services { + routes = append(routes, service.Routes...) + } + + return &translator.TranslateResult{ + Routes: routes, + Services: resources.Services, + SSL: resources.SSLs, + GlobalRules: resources.GlobalRules, + PluginMetadata: resources.PluginMetadata, + Consumers: resources.Consumers, + }, nil +} + +// adcRouteResource implements RouteResource +type adcRouteResource struct { + *adcDataplaneResource +} + +func (r *adcRouteResource) List(ctx context.Context) ([]*adctypes.Route, error) { + result, err := r.dumpResources(ctx) + if err != nil { + return nil, err + } + return result.Routes, nil +} + +// adcServiceResource implements ServiceResource +type adcServiceResource struct { + *adcDataplaneResource +} + +func (s *adcServiceResource) List(ctx context.Context) ([]*adctypes.Service, error) { + result, err := s.dumpResources(ctx) + if err != nil { + return nil, err + } + return result.Services, nil +} + +// adcSSLResource implements SSLResource +type adcSSLResource struct { + *adcDataplaneResource +} + +func (s *adcSSLResource) List(ctx context.Context) ([]*adctypes.SSL, error) { + result, err := s.dumpResources(ctx) + if err != nil { + return nil, err + } + return result.SSL, nil +} + +// adcConsumerResource implements ConsumerResource +type adcConsumerResource struct { + *adcDataplaneResource +} + +func (c *adcConsumerResource) List(ctx context.Context) ([]*adctypes.Consumer, error) { + result, err := c.dumpResources(ctx) + if err != nil { + return nil, err + } + return result.Consumers, nil +} diff --git a/test/e2e/scaffold/api7_deployer.go b/test/e2e/scaffold/api7_deployer.go index b50458192..5415d2ab0 100644 --- a/test/e2e/scaffold/api7_deployer.go +++ b/test/e2e/scaffold/api7_deployer.go @@ -13,7 +13,6 @@ package scaffold import ( - "context" "fmt" "os" "time" @@ -24,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/apache/apisix-ingress-controller/pkg/dashboard" "github.com/apache/apisix-ingress-controller/pkg/utils" "github.com/apache/apisix-ingress-controller/test/e2e/framework" ) @@ -90,7 +88,6 @@ func (s *API7Deployer) BeforeEach() { e.Add(func() { s.DeployDataplane() s.DeployIngress() - s.initDataPlaneClient() }) e.Add(s.DeployTestService) e.Wait() @@ -181,34 +178,6 @@ func (s *API7Deployer) ScaleIngress(replicas int) { }) } -func (s *API7Deployer) initDataPlaneClient() { - var err error - s.apisixCli, err = dashboard.NewClient() - Expect(err).NotTo(HaveOccurred(), "creating apisix client") - - adminURL := fmt.Sprintf("http://%s/apisix/admin", s.GetDashboardEndpoint()) - - s.Logf("apisix admin: %s", adminURL) - - err = s.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ - Name: "default", - ControllerName: s.opts.ControllerName, - Labels: map[string]string{"k8s/controller-name": s.opts.ControllerName}, - BaseURL: adminURL, - AdminKey: s.AdminKey(), - }) - Expect(err).NotTo(HaveOccurred(), "adding cluster") - - httpsURL := fmt.Sprintf("https://%s/apisix/admin", s.GetDashboardEndpointHTTPS()) - err = s.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ - Name: "default-https", - BaseURL: httpsURL, - AdminKey: s.AdminKey(), - SkipTLSVerify: true, - }) - Expect(err).NotTo(HaveOccurred(), "adding cluster") -} - // CreateAdditionalGateway creates a new gateway group and deploys a dataplane for it. // It returns the gateway group ID and namespace name where the dataplane is deployed. func (s *API7Deployer) CreateAdditionalGateway(namePrefix string) (string, string, error) { @@ -303,3 +272,12 @@ func (s *API7Deployer) GetAdminEndpoint(_ ...*corev1.Service) string { // always return the default dashboard endpoint return framework.DashboardTLSEndpoint } + +func (s *API7Deployer) DefaultDataplaneResource() DataplaneResource { + return newADCDataplaneResource( + "api7ee", + fmt.Sprintf("http://%s", s.GetDashboardEndpoint()), + s.AdminKey(), + false, + ) +} diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index 6a8e3cd56..1ada35d22 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -40,6 +40,7 @@ type APISIXDeployOptions struct { type APISIXDeployer struct { *Scaffold + adminTunnel *k8s.Tunnel } func NewAPISIXDeployer(s *Scaffold) *APISIXDeployer { @@ -91,6 +92,9 @@ func (s *APISIXDeployer) BeforeEach() { e.Add(func() { s.DeployDataplane() s.DeployIngress() + adminTunnel, err := s.createAdminTunnel(s.dataplaneService) + Expect(err).NotTo(HaveOccurred()) + s.adminTunnel = adminTunnel }) e.Add(s.DeployTestService) e.Wait() @@ -220,12 +224,7 @@ func getEnvOrDefault(key, defaultValue string) string { return defaultValue } -//nolint:unused -func (s *APISIXDeployer) createAdminTunnel( - svc *corev1.Service, - kubectlOpts *k8s.KubectlOptions, - serviceName string, -) (*k8s.Tunnel, error) { +func (s *APISIXDeployer) createAdminTunnel(svc *corev1.Service) (*k8s.Tunnel, error) { var ( adminNodePort int adminPort int @@ -239,7 +238,9 @@ func (s *APISIXDeployer) createAdminTunnel( } } - adminTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + kubectlOpts := k8s.NewKubectlOptions("", "", svc.Namespace) + + adminTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, svc.Name, adminNodePort, adminPort) if err := adminTunnel.ForwardPortE(s.t); err != nil { @@ -343,3 +344,12 @@ func (s *APISIXDeployer) GetAdminEndpoint(svc ...*corev1.Service) string { } return fmt.Sprintf("http://%s.%s:9180", svc[0].Name, svc[0].Namespace) } + +func (s *APISIXDeployer) DefaultDataplaneResource() DataplaneResource { + return newADCDataplaneResource( + "apisix-standalone", + fmt.Sprintf("http://%s", s.adminTunnel.Endpoint()), + s.AdminKey(), + false, // tlsVerify + ) +} diff --git a/test/e2e/scaffold/deployer.go b/test/e2e/scaffold/deployer.go index 75614ed5d..f4216d896 100644 --- a/test/e2e/scaffold/deployer.go +++ b/test/e2e/scaffold/deployer.go @@ -12,7 +12,9 @@ package scaffold -import corev1 "k8s.io/api/core/v1" +import ( + corev1 "k8s.io/api/core/v1" +) // Deployer defines the interface for deploying data plane components type Deployer interface { @@ -25,6 +27,7 @@ type Deployer interface { CreateAdditionalGateway(namePrefix string) (string, string, error) CleanupAdditionalGateway(identifier string) error GetAdminEndpoint(...*corev1.Service) string + DefaultDataplaneResource() DataplaneResource } var NewDeployer func(*Scaffold) Deployer diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 3779b3ed2..2121bd897 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -29,7 +29,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/apache/apisix-ingress-controller/pkg/dashboard" "github.com/apache/apisix-ingress-controller/test/e2e/framework" ) @@ -66,9 +65,6 @@ type Scaffold struct { additionalGateways map[string]*GatewayResources - // TODO: move to deployer - apisixCli dashboard.Dashboard - Deployer Deployer } @@ -191,16 +187,8 @@ func (s *Scaffold) NewAPISIXHttpsClient(host string) *httpexpect.Expect { }) } -func (s *Scaffold) DefaultDataplaneResource() dashboard.Cluster { - return s.apisixCli.Cluster("default") -} - -func (s *Scaffold) DefaultDataplaneResourceHTTPS() dashboard.Cluster { - return s.apisixCli.Cluster("default-https") -} - -func (s *Scaffold) DataPlaneClient() dashboard.Dashboard { - return s.apisixCli +func (s *Scaffold) DefaultDataplaneResource() DataplaneResource { + return s.Deployer.DefaultDataplaneResource() } func (s *Scaffold) DeployTestService() {