diff --git a/Makefile b/Makefile index 229885af0..18ed521fc 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ VERSION ?= 2.0.0 +RELEASE_SRC = apache-apisix-ingress-controller-${VERSION}-src IMAGE_TAG ?= dev @@ -31,14 +32,17 @@ GATEAY_API_VERSION ?= v1.2.0 DASHBOARD_VERSION ?= dev ADC_VERSION ?= 0.20.0 +GINKGO_VERSION ?= 2.20.0 TEST_TIMEOUT ?= 80m TEST_DIR ?= ./test/e2e/ +E2E_NODES ?= 2 # CRD Reference Documentation CRD_REF_DOCS_VERSION ?= v0.1.0 CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs -CRD_DOCS_CONFIG ?= docs/crd/config.yaml -CRD_DOCS_OUTPUT ?= docs/crd/api.md +CRD_DOCS_CONFIG ?= docs/assets/crd/config.yaml +CRD_DOCS_OUTPUT ?= docs/en/latest/api-reference.md +CRD_DOCS_TEMPLATE ?= docs/assets/template export KUBECONFIG = /tmp/$(KIND_NAME).kubeconfig @@ -137,6 +141,14 @@ download-api7ee3-chart: @helm pull api7/api7ee3 --destination "$(shell helm env HELM_REPOSITORY_CACHE)" @echo "Downloaded API7EE3 chart" +.PHONY: ginkgo-e2e-test +ginkgo-e2e-test: + @ginkgo -cover -coverprofile=coverage.txt -r --randomize-all --randomize-suites --trace --focus=$(E2E_FOCUS) --nodes=$(E2E_NODES) $(TEST_DIR) + +.PHONY: install-ginkgo +install-ginkgo: + @go install github.com/onsi/ginkgo/v2/ginkgo@v$(GINKGO_VERSION) + .PHONY: conformance-test conformance-test: DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance -tags=conformance -timeout 60m @@ -170,15 +182,15 @@ kind-down: .PHONY: kind-load-images kind-load-images: pull-infra-images kind-load-ingress-image - @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-gateway:dev --name $(KIND_NAME) - @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-dp-manager:$(DASHBOARD_VERSION) --name $(KIND_NAME) - @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-integrated:$(DASHBOARD_VERSION) --name $(KIND_NAME) - @kind load docker-image kennethreitz/httpbin:latest --name $(KIND_NAME) + @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-gateway:dev --name $(KIND_NAME) + @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-dp-manager:$(DASHBOARD_VERSION) --name $(KIND_NAME) + @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-integrated:$(DASHBOARD_VERSION) --name $(KIND_NAME) + @kind load docker-image kennethreitz/httpbin:latest --name $(KIND_NAME) @kind load docker-image jmalloc/echo-server:latest --name $(KIND_NAME) .PHONY: kind-load-gateway-image kind-load-gateway-image: - @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-gateway:dev --name $(KIND_NAME) + @kind load docker-image hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-gateway:dev --name $(KIND_NAME) .PHONY: kind-load-dashboard-images kind-load-dashboard-images: @@ -192,7 +204,7 @@ kind-load-ingress-image: .PHONY: pull-infra-images pull-infra-images: @docker pull hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-gateway:dev - @docker pull hkccr.ccs.tencentyun.com/api7-dev/api7-ee-dp-manager:$(DASHBOARD_VERSION) + @docker pull hkccr.ccs.tencentyun.com/api7-dev/api7-ee-dp-manager:$(DASHBOARD_VERSION) @docker pull hkccr.ccs.tencentyun.com/api7-dev/api7-ee-3-integrated:$(DASHBOARD_VERSION) @docker pull kennethreitz/httpbin:latest @docker pull jmalloc/echo-server:latest @@ -389,7 +401,7 @@ generate-crd-docs: manifests ## Generate CRD reference documentation in a single --source-path=./api \ --config=$(CRD_DOCS_CONFIG) \ --renderer=markdown \ - --templates-dir=./docs/template \ + --templates-dir=$(CRD_DOCS_TEMPLATE) \ --output-path=$(CRD_DOCS_OUTPUT) \ --max-depth=100 @echo "CRD reference documentation generated at $(CRD_DOCS_OUTPUT)" @@ -402,7 +414,7 @@ generate-crd-docs-grouped: manifests ## Generate CRD reference documentation gro --source-path=./api \ --config=$(CRD_DOCS_CONFIG) \ --renderer=markdown \ - --templates-dir=./docs/template \ + --templates-dir=$(CRD_DOCS_TEMPLATE) \ --output-path=docs/crd/groups \ --output-mode=group @echo "CRD reference documentation generated in docs/crd/groups directory" @@ -416,3 +428,56 @@ verify-license: .PHONY: update-license update-license: docker run -it --rm -v $(PWD):/github/workspace apache/skywalking-eyes header fix + +### verify-mdlint: Verify markdown files lint rules. +.PHONY: verify-mdlint +verify-mdlint: + docker run -it --rm -v $(PWD):/work tmknom/markdownlint '**/*.md' --ignore node_modules --ignore CHANGELOG.md + +### update-mdlint: Update markdown files lint rules. +.PHONY: update-mdlint +update-mdlint: + docker run -it --rm -v $(PWD):/work tmknom/markdownlint '**/*.md' -f --ignore node_modules --ignore vendor --ignore CHANGELOG.md + +### verify-yamllint: Verify yaml files lint rules for `examples` directory. +.PHONY: verify-yamllint +verify-yamllint: + docker run -it --rm -v $(PWD):/yaml peterdavehello/yamllint yamllint examples + +### update-yamlfmt: Update yaml files format for `examples` directory. +.PHONY: update-yamlfmt +update-yamlfmt: + go install github.com/google/yamlfmt/cmd/yamlfmt@latest && yamlfmt examples + +### verify-all: Verify all verify- rules. +.PHONY: verify-all +verify-all: verify-license verify-mdlint verify-yamllint + +### update-all: Update all update- rules. +.PHONY: update-all +update-all: update-license update-mdlint update-gofmt + +### release-src: Release source +release-src: + tar -zcvf $(RELEASE_SRC).tgz \ + --exclude .github \ + --exclude .git \ + --exclude .idea \ + --exclude .gitignore \ + --exclude .DS_Store \ + --exclude docs \ + --exclude examples \ + --exclude scripts \ + --exclude samples \ + --exclude test \ + --exclude release \ + --exclude $(RELEASE_SRC).tgz \ + . + + gpg --batch --yes --armor --detach-sig $(RELEASE_SRC).tgz + shasum -a 512 $(RELEASE_SRC).tgz > $(RELEASE_SRC).tgz.sha512 + + mkdir -p release + mv $(RELEASE_SRC).tgz release/$(RELEASE_SRC).tgz + mv $(RELEASE_SRC).tgz.asc release/$(RELEASE_SRC).tgz.asc + mv $(RELEASE_SRC).tgz.sha512 release/$(RELEASE_SRC).tgz.sha512 diff --git a/docs/crd/config.yaml b/docs/assets/crd/config.yaml similarity index 100% rename from docs/crd/config.yaml rename to docs/assets/crd/config.yaml diff --git a/docs/assets/images/api7-ingress-controller-architecture.png b/docs/assets/images/api7-ingress-controller-architecture.png deleted file mode 100644 index 089419160..000000000 Binary files a/docs/assets/images/api7-ingress-controller-architecture.png and /dev/null differ diff --git a/docs/assets/images/gateway-api-extensions-resources.png b/docs/assets/images/gateway-api-extensions-resources.png new file mode 100644 index 000000000..cc618ff2c Binary files /dev/null and b/docs/assets/images/gateway-api-extensions-resources.png differ diff --git a/docs/assets/images/ingress-admin-api-architecture.png b/docs/assets/images/ingress-admin-api-architecture.png new file mode 100644 index 000000000..91cc07c0b Binary files /dev/null and b/docs/assets/images/ingress-admin-api-architecture.png differ diff --git a/docs/assets/images/ingress-standalone-architecture.png b/docs/assets/images/ingress-standalone-architecture.png new file mode 100644 index 000000000..9c557eb1f Binary files /dev/null and b/docs/assets/images/ingress-standalone-architecture.png differ diff --git a/docs/assets/images/v2-crds-api-resources.png b/docs/assets/images/v2-crds-api-resources.png new file mode 100644 index 000000000..d5f3c21d7 Binary files /dev/null and b/docs/assets/images/v2-crds-api-resources.png differ diff --git a/docs/template/gv_details.tpl b/docs/assets/template/gv_details.tpl similarity index 96% rename from docs/template/gv_details.tpl rename to docs/assets/template/gv_details.tpl index eae500cf7..611ca1c9b 100644 --- a/docs/template/gv_details.tpl +++ b/docs/assets/template/gv_details.tpl @@ -39,7 +39,7 @@ ### Types -This section describes the types used by the CRDs. +In this section you will find types that the CRDs rely on. {{- /* Display Types that are not exported Kinds */ -}} {{- range $typ := $gv.SortedTypes -}} diff --git a/docs/template/gv_list.tpl b/docs/assets/template/gv_list.tpl similarity index 91% rename from docs/template/gv_list.tpl rename to docs/assets/template/gv_list.tpl index d8a37d274..d723151aa 100644 --- a/docs/template/gv_list.tpl +++ b/docs/assets/template/gv_list.tpl @@ -26,7 +26,7 @@ slug: /reference/apisix-ingress-controller/crd-reference description: Explore detailed reference documentation for the custom resource definitions (CRDs) supported by the APISIX Ingress Controller. --- -This document provides the API resource description for the API7 Ingress Controller custom resource definitions (CRDs). +This document provides the API resource description the APISIX Ingress Controller custom resource definitions (CRDs). ## Packages {{- range $groupVersions }} diff --git a/docs/template/type.tpl b/docs/assets/template/type.tpl similarity index 100% rename from docs/template/type.tpl rename to docs/assets/template/type.tpl diff --git a/docs/template/type_members.tpl b/docs/assets/template/type_members.tpl similarity index 100% rename from docs/template/type_members.tpl rename to docs/assets/template/type_members.tpl diff --git a/docs/concepts.md b/docs/concepts.md deleted file mode 100644 index c320753a6..000000000 --- a/docs/concepts.md +++ /dev/null @@ -1,27 +0,0 @@ -# Concepts - -The APISIX Ingress Controller is used to manage the APISIX Gateway as either a standalone application or a Kubernetes-based application. It dynamically configures and manages the API7 Gateway using Gateway API resources. - -## Architecture - -![APISIX Ingress Controller Architecture](./assets/images/api7-ingress-controller-architecture.png) - -## Kubernetes Resources - -### Service - -In Kubernetes, a Service is a method to expose network applications running on a set of Pods as network services. - -When proxying ingress traffic, APISIX Gateway by default directs traffic directly to the Pods instead of through kube-proxy. - -### EndpointSlicea - -EndpointSlice objects represent subsets (slices) of backend network endpoints for a Service. - -The APISIX Ingress Controller continuously tracks matching EndpointSlice objects, and whenever the set of Pods in a Service changes, the set of Pods proxied by the APISIX Gateway will also update accordingly. - -## Gateway API - -Gateway API is an official Kubernetes project focused on L4 and L7 routing in Kubernetes. This project represents the next generation of Kubernetes Ingress, Load Balancing, and Service Mesh APIs. - -For more information on supporting Gateway API, please refer to [Gateway API](./gateway-api.md). diff --git a/docs/crd/api.md b/docs/en/latest/api-reference.md similarity index 99% rename from docs/crd/api.md rename to docs/en/latest/api-reference.md index b0d47f256..11fce5b36 100644 --- a/docs/crd/api.md +++ b/docs/en/latest/api-reference.md @@ -4,7 +4,7 @@ slug: /reference/apisix-ingress-controller/crd-reference description: Explore detailed reference documentation for the custom resource definitions (CRDs) supported by the APISIX Ingress Controller. --- -This document provides the API resource description for the API7 Ingress Controller custom resource definitions (CRDs). +This document provides the API resource description the APISIX Ingress Controller custom resource definitions (CRDs). ## Packages - [apisix.apache.org/v1alpha1](#apisixapacheorgv1alpha1) @@ -102,7 +102,7 @@ PluginConfig defines plugin configuration. ### Types -This section describes the types used by the CRDs. +In this section you will find types that the CRDs rely on. #### AdminKeyAuth @@ -607,7 +607,7 @@ ApisixUpstream defines configuration for upstream services. ### Types -This section describes the types used by the CRDs. +In this section you will find types that the CRDs rely on. #### ActiveHealthCheck diff --git a/docs/en/latest/concepts/deployment-architecture.md b/docs/en/latest/concepts/deployment-architecture.md new file mode 100644 index 000000000..e8b5c9b00 --- /dev/null +++ b/docs/en/latest/concepts/deployment-architecture.md @@ -0,0 +1,42 @@ +--- +title: Deployment Architecture +keywords: + - APISIX Ingress + - Apache APISIX + - Kubernetes Ingress + - Gateway API +--- + + +The APISIX Ingress Controller is used to manage the APISIX Gateway as either a standalone application or a Kubernetes-based application. It dynamically configures and manages the APISIX Gateway using Gateway API resources. + +## Admin API Mode + +In the traditional deployment approach, APISIX uses etcd as its configuration center, allowing administrators to dynamically manage routes, upstreams, and other resources through RESTful APIs. It supports distributed cluster deployments with real-time configuration synchronization. + +![Admin API Architecture](../../../assets/images/ingress-admin-api-architecture.png) + +## Standalone Mode (Experimental) + +APISIX runs independently without relying on etcd, supporting two sub-modes - file-driven (managing configuration through conf/apisix.yaml files) and API-driven (storing configuration in memory with full configuration management through the dedicated /apisix/admin/configs endpoint). + +This mode is particularly suitable for Kubernetes environments and single-node deployments, where the API-driven memory management approach combines the convenience of traditional Admin API with the simplicity of Standalone mode. + +![Standalone Architecture](../../../assets/images/ingress-standalone-architecture.png) diff --git a/docs/gateway-api.md b/docs/en/latest/concepts/gateway-api.md similarity index 76% rename from docs/gateway-api.md rename to docs/en/latest/concepts/gateway-api.md index 459ff5a83..44f5376f1 100644 --- a/docs/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -1,5 +1,29 @@ - -# Gateway API +--- +title: Gateway API +keywords: + - APISIX Ingress + - Apache APISIX + - Kubernetes Ingress + - Gateway API +--- + Gateway API is dedicated to achieving expressive and scalable Kubernetes service networking through various custom resources. @@ -29,7 +53,7 @@ For more information about Gateway API, please refer to [Gateway API](https://ga ## HTTPRoute -The HTTPRoute resource allows users to configure HTTP routing by matching HTTP traffic and forwarding it to Kubernetes backends. Currently, the only backend supported by API7 Gateway is the Service resource. +The HTTPRoute resource allows users to configure HTTP routing by matching HTTP traffic and forwarding it to Kubernetes backends. Currently, the only backend supported by APISIX Gateway is the Service resource. ### Example diff --git a/docs/en/latest/concepts/resources.md b/docs/en/latest/concepts/resources.md new file mode 100644 index 000000000..df1a06816 --- /dev/null +++ b/docs/en/latest/concepts/resources.md @@ -0,0 +1,87 @@ +--- +title: APISIX Ingress Controller Resources +keywords: + - APISIX Ingress + - Apache APISIX + - Kubernetes Ingress + - Gateway API +description: APISIX Ingress Controller Resources, including Kubernetes resources, Gateway API, and APISIX Ingress Controller CRDs API. +--- + + +## Kubernetes Resources + +### Service + +In Kubernetes, a Service is a method to expose network applications running on a set of Pods as network services. + +When proxying ingress traffic, APISIX Gateway by default directs traffic directly to the Pods instead of through kube-proxy. + +### EndpointSlices + +EndpointSlice objects represent subsets (slices) of backend network endpoints for a Service. + +The APISIX Ingress Controller continuously tracks matching EndpointSlice objects, and whenever the set of Pods in a Service changes, the set of Pods proxied by the APISIX Gateway will also update accordingly. + +### Ingress + +Ingress is a Kubernetes resource that manages external access to services within a cluster, typically HTTP and HTTPS traffic. It provides a way to define rules for routing external traffic to internal services. + +## Gateway API + +Gateway API is an official Kubernetes project focused on L4 and L7 routing in Kubernetes. This project represents the next generation of Kubernetes Ingress, Load Balancing, and Service Mesh APIs. + +For more information on supporting Gateway API, please refer to [Gateway API](./gateway-api.md). + +## APISIX Ingress Controller CRDs API + +The APISIX Ingress Controller defines several Custom Resource Definitions (CRDs) to manage routing, upstreams, TLS, and cluster settings declaratively. + +### Gateway API Extensions + +Enable additional features not included in the standard Kubernetes Gateway API, developed and maintained by Gateway API implementers to extend functionality securely and reliably. + +* GatewayProxy: Defines connection settings between the APISIX Ingress Controller and APISIX, including auth, endpoints, and global plugins. Referenced via parametersRef in Gateway, GatewayClass, or IngressClass + +* BackendTrafficPolicy: Defines traffic management settings for backend services, including load balancing, timeouts, retries, and host header handling in the APISIX Ingress Controller. + +* Consumer: Defines API consumers and their credentials, enabling authentication and plugin configuration for controlling access to API endpoints. + +* PluginConfig: Defines reusable plugin configurations that can be referenced by other resources like HTTPRoute, enabling separation of routing logic and plugin settings for better reusability and manageability. + +* HTTPRoutePolicy: Configures advanced traffic management and routing policies for HTTPRoute or Ingress resources, enhancing functionality without modifying the original resources. + +![Gateway API Extensions Overview](../../../assets/images/gateway-api-extensions-resources.png) + +### Ingress API Extensions + +APISIX Ingress Controller CRDs extend Kubernetes functionality to provide declarative configuration management for the Apache APISIX gateway, supporting advanced routing, traffic management, and security policies. + +* ApisixRoute: Defines routing rules for HTTP/TCP/UDP, supporting path matching, hostnames, method filtering, and backend service configurations. Can reference ApisixUpstream and ApisixPluginConfig resources. + +* ApisixUpstream: Extends Kubernetes Services with advanced configurations such as load balancing, health checks, retries, timeouts, and service subset selection. + +* ApisixConsumer: Defines API consumers and their authentication credentials, supporting methods like basicAuth, keyAuth, jwtAuth, hmacAuth, wolfRBAC, and ldapAuth. + +* ApisixPluginConfig: Defines reusable plugin configurations referenced by ApisixRoute through the plugin_config_name field, promoting separation of routing logic and plugin settings. + +* ApisixTls: Manages SSL/TLS certificates, supporting SNI binding and mutual TLS for secure APISIX gateway connections. + +![V2 CRDs Overview](../../../assets/images/v2-crds-api-resources.png) diff --git a/docs/configure.md b/docs/en/latest/configure.md similarity index 56% rename from docs/configure.md rename to docs/en/latest/configure.md index 06c69d5da..c4a104e90 100644 --- a/docs/configure.md +++ b/docs/en/latest/configure.md @@ -1,4 +1,30 @@ -# Configure +--- +title: Configuration +keywords: + - APISIX Ingress + - Apache APISIX + - Kubernetes Ingress + - Gateway API +description: Configuration of the APISIX Ingress Controller +--- + The APISIX Ingress Controller is a Kubernetes Ingress Controller that implements the Gateway API. This document describes how to configure the APISIX Ingress Controller. diff --git a/docs/en/latest/developer-guide.md b/docs/en/latest/developer-guide.md new file mode 100644 index 000000000..90fdacc78 --- /dev/null +++ b/docs/en/latest/developer-guide.md @@ -0,0 +1,118 @@ +--- +title: Developer Guide +keywords: + - APISIX ingress + - Apache APISIX + - Kubernetes ingress + - Development + - Contribute +description: Setting up development environment for APISIX Ingress controller. +--- + + +This document walks through how you can set up your development environment to contribute to APISIX Ingress controller. + +## Prerequisites + +Before you get started make sure you have: + +1. Installed [Go 1.23](https://golang.org/dl/) or later +2. A Kubernetes cluster available. We recommend using [kind](https://kind.sigs.k8s.io/). +3. Installed APISIX in Kubernetes using [Helm](https://github.com/apache/apisix-helm-chart). +4. Installed [ADC v0.20.0+](https://github.com/api7/adc/releases) + +## Fork and clone + +1. Fork the repository [apache/apisix-ingress-controller](https://github.com/apache/apisix-ingress-controller) to your GitHub account +2. Clone the fork to your workstation. +3. Run `go mod download` to download the required modules. + +:::tip + +If you are in China, you can speed up the downloads by setting `GOPROXY` to `https://goproxy.cn`. + +::: + +## Install CRD and Gateway API + +To install the [CRD](./concepts/resources.md#apisix-ingress-controller-crds-api) and [Gateway API](https://gateway-api.sigs.k8s.io/), run the following commands: + +```shell +make install +``` + +## Build from source + +To build APISIX Ingress controller, run the command below on the root of the project: + +```shell +make build +``` + +Now you can run it by: + +```shell +# for ARM64 architecture, use the following command: +# ./bin/apisix-ingress-controller_arm64 version +./bin/apisix-ingress-controller_amd64 version +``` + +## Building Image + +To build a Docker image for APISIX Ingress controller, you can use the following command: + +```shell +make build-image IMG=apache/apisix-ingress-controller:dev +``` + +## Running tests + +### Unit Tests + +To run unit tests: + +```shell +make unit-test +``` + +### e2e Tests + +To run end-to-end tests, you need to install [kind](https://kind.sigs.k8s.io/). + +Launch a kind cluster with the following command: + +```shell +make kind-up +``` + +To run end-to-end e2e-tests against any changes, you need to load the built Docker images into the Kubernetes cluster: + +```shell +# build docker image for APISIX Ingress controller +make build-image +# load the image into kind cluster +make kind-load-images +``` + +Currently, we use Kind version `0.26.0` and Kubernetes version `1.26+` for running the tests. + +```shell +make e2e-test +``` diff --git a/docs/getting-started.md b/docs/en/latest/getting-started.md similarity index 91% rename from docs/getting-started.md rename to docs/en/latest/getting-started.md index 3f20bc2c5..fc8d02137 100644 --- a/docs/getting-started.md +++ b/docs/en/latest/getting-started.md @@ -43,9 +43,9 @@ Before installing APISIX Ingress Controller, ensure you have: 1. A working Kubernetes cluster (version 1.26+) 2. [Helm](https://helm.sh/) (version 3.8+) installed -### Install APISIX and APISIX Ingress Controller +### Install APISIX and APISIX Ingress Controller (Standalone API-driven mode) -Install the Gateway API CRDs, APISIX, and APISIX Ingress Controller using the following commands: +Install the Gateway API CRDs, [APISIX Standalone API-driven mode](https://apisix.apache.org/docs/apisix/deployment-modes/#api-driven-experimental), and APISIX Ingress Controller using the following commands: ```bash helm repo add apisix https://charts.apiseven.com @@ -55,7 +55,11 @@ helm repo update helm install apisix \ --namespace ingress-apisix \ --create-namespace \ + --set apisix.deployment.role=traditional \ + --set apisix.deployment.role_traditional.config_provider=yaml \ + --set etcd.enabled=false \ --set ingress-controller.enabled=true \ + --set ingress-controller.config.provider.type=apisix-standalone \ --set ingress-controller.apisix.adminService.namespace=ingress-apisix \ --set ingress-controller.gatewayProxy.createDefault=true \ apisix/apisix @@ -66,7 +70,7 @@ helm install apisix \ Install the httpbin example application to test the configuration: ```bash -https://raw.githubusercontent.com/apache/apisix-ingress-controller/refs/heads/v2.0.0/examples/httpbin/deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/apache/apisix-ingress-controller/refs/heads/v2.0.0/examples/httpbin/deployment.yaml ``` ### Configure a Route diff --git a/docs/upgrade-guide.md b/docs/en/latest/upgrade-guide.md similarity index 70% rename from docs/upgrade-guide.md rename to docs/en/latest/upgrade-guide.md index 297fff7b3..6b3fa7f94 100644 --- a/docs/upgrade-guide.md +++ b/docs/en/latest/upgrade-guide.md @@ -1,4 +1,29 @@ -# APISIX Ingress Controller Upgrade Guide +--- +title: Upgrade Guide +keywords: + - APISIX Ingress + - Apache APISIX + - Kubernetes Ingress + - Gateway API +--- + ## Upgrading from 1.x.x to 2.0.0: Key Changes and Considerations @@ -21,7 +46,7 @@ There were two main deployment architectures in 1.x.x: #### Architecture in 2.0.0 -![upgrade to 2.0.0 architecture](./assets/images/upgrade-to-architecture.png) +![upgrade to 2.0.0 architecture](../../assets/images/upgrade-to-architecture.png) ##### Mock-ETCD Mode Deprecated @@ -39,7 +64,13 @@ etcdserver: ##### Controller-Only Configuration Source -In 2.0.0, all data plane configurations must originate from the Ingress Controller. Configurations via Admin API or any external methods are no longer supported and will be ignored or may cause errors. +Starting with APISIX Ingress Controller 2.0.0, the controller is the single source of truth. Manual Admin API changes will be overwritten on the next full sync. The prior approach, which allowed controller-managed and manually added configurations to coexist, was incorrect and is now deprecated. + +#### APISIX With Etcd (Admin API) synchronization performance + +In APISIX Ingress Controller 2.0.0, ADC performs scheduled resource synchronization by comparing resources against the admin API response. + +Because the Admin API fills in default values, the submitted content may differ from the returned result. This breaks the diff, triggering full updates to data plane resources, causing cache invalidation and significant performance impact. ### Ingress Configuration Changes @@ -108,14 +139,24 @@ spec: #### `ApisixUpstream` -Due to current limitations in the ADC (API Definition Controller) component, the following fields are not yet supported: +Due to current limitations in the [ADC](https://github.com/api7/adc) component, the following fields are not yet supported: * `spec.discovery`: Service Discovery * `spec.healthCheck`: Health Checking More details: [ADC Backend Differences](https://github.com/api7/adc/blob/2449ca81e3c61169f8c1e59efb4c1173a766bce2/libs/backend-apisix-standalone/README.md#differences-in-upstream) -#### Limited Support for Ingress Annotations +#### `ApisixClusterConfig` + +The `ApisixClusterConfig` CRD has been removed in 2.0.0. global rules and configurations should now be managed through the `ApisixGlobalRule` CRDs. + +#### Ingress + +##### API Version Support + +Currently supports networking.k8s.io/v1 only. Support for other Ingress API versions (networking.k8s.io/v1beta1 and extensions/v1beta1) is not yet available in 2.0.0. + +##### Limited Support for Ingress Annotations Ingress annotations used in version 1.x.x are not fully supported in 2.0.0. If your existing setup relies on any of the following annotations, validate compatibility or consider delaying the upgrade. @@ -162,10 +203,10 @@ Ingress annotations used in version 1.x.x are not fully supported in 2.0.0. If y ### Summary -| Category | Description | -| ---------------- | ---------------------------------------------------------------------------------------------------- | -| Architecture | The `mock-etcd` component has been removed. Configuration is now centralized through the Controller. | -| Configuration | Static configuration fields have been removed. Use `GatewayProxy` CRD to configure the data plane. | -| Data Plane | Requires APISIX version 3.13.0 running in `standalone` mode. | -| API | Some fields in `Ingress Annotations` and `ApisixUpstream` are not yet supported. | -| Upgrade Strategy | Blue-green deployment or canary release is recommended before full switchover. | +| Category | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Architecture | The `mock-etcd` component has been removed. Configuration is now centralized through the Controller. | +| Configuration | Static configuration fields have been removed. Use `GatewayProxy` CRD to configure the data plane. | +| Data Plane | The Admin API configuration method is still supported. Support for the Standalone API-driven mode was introduced in APISIX 3.13.0 and later. | +| API | Some fields in `Ingress Annotations` and `ApisixUpstream` are not yet supported. | +| Upgrade Strategy | Blue-green deployment or canary release is recommended before full switchover. | diff --git a/docs/quickstart.md b/docs/quickstart.md deleted file mode 100644 index da0a3a686..000000000 --- a/docs/quickstart.md +++ /dev/null @@ -1,60 +0,0 @@ -# Quickstart - -This quickstart guide will help you get started with APISIX Ingress Controller in a few simple steps. - -## Prerequisites - -* Kubernetes -* API7 Dashboard -* API7 Gateway - -Please ensure you have deployed the API7 Dashboard control plane. - -Note: Refer to the [Gateway API Release Changelog](https://github.com/kubernetes-sigs/gateway-api/releases/tag/v1.0.0), it is recommended to use Kubernetes version 1.25+. - -## Installation - -Install the Gateway API CRDs: - -```shell -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml - -``` - -Install The APISIX Ingress Controller: - -```shell -kubectl apply -f https://github.com/apache/apisix-ingress-controller/releases/download/install.yaml - -``` - -## Test HTTP Routing - -Install the GatewayClass, Gateway, HTTPRoute and httpbin example app: - -```shell -kubectl apply -f https://github.com/apache/apisix-ingress-controller/blob/release-v2-dev/examples/quickstart.yaml -``` - -Requests will be forwarded by the gateway to the httpbin application: - -```shell -curl http://{apisix_gateway_loadbalancer_ip}/headers -``` - -:::Note If the APISIX Gateway service without loadbalancer - -You can forward the local port to the APISIX Gateway service with the following command: - -```shell -# Listen on port 9080 locally, forwarding to 80 in the pod -kubectl port-forward svc/${apisix-gateway-svc} 9080:80 -n ${apisix_gateway_namespace} -``` - -Now you can send HTTP requests to access it: - -```shell -curl http://localhost:9080/headers -``` - -::: diff --git a/internal/manager/run.go b/internal/manager/run.go index 54ee31c8b..b48746c16 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -21,7 +21,6 @@ import ( "context" "crypto/tls" "os" - "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -172,36 +171,6 @@ func Run(ctx context.Context, logger logr.Logger) error { return err } - go func() { - setupLog.Info("starting provider sync") - initalSyncDelay := config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration - time.AfterFunc(initalSyncDelay, func() { - setupLog.Info("trying to initialize provider") - if err := provider.Sync(ctx); err != nil { - setupLog.Error(err, "unable to sync resources to provider") - return - } - }) - - syncPeriod := config.ControllerConfig.ProviderConfig.SyncPeriod.Duration - if syncPeriod < 1 { - return - } - ticker := time.NewTicker(syncPeriod) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := provider.Sync(ctx); err != nil { - setupLog.Error(err, "unable to sync resources to provider") - return - } - case <-ctx.Done(): - return - } - } - }() - setupLog.Info("check ReferenceGrants is enabled") _, err = mgr.GetRESTMapper().KindsFor(schema.GroupVersionResource{ Group: v1beta1.GroupVersion.Group, diff --git a/internal/provider/adc/cache/indexer.go b/internal/provider/adc/cache/indexer.go index e19410b84..ad9dda3bd 100644 --- a/internal/provider/adc/cache/indexer.go +++ b/internal/provider/adc/cache/indexer.go @@ -56,6 +56,12 @@ type LabelIndexer struct { GetLabels func(obj any) map[string]string } +// ref: https://pkg.go.dev/github.com/hashicorp/go-memdb#Txn.Get +// by adding suffixes to avoid prefix matching +func (emi *LabelIndexer) genKey(labelValues []string) []byte { + return []byte(strings.Join(labelValues, "/") + "\x00") +} + func (emi *LabelIndexer) FromObject(obj any) (bool, []byte, error) { labels := emi.GetLabels(obj) var labelValues []string @@ -69,7 +75,7 @@ func (emi *LabelIndexer) FromObject(obj any) (bool, []byte, error) { return false, nil, nil } - return true, []byte(strings.Join(labelValues, "/")), nil + return true, emi.genKey(labelValues), nil } func (emi *LabelIndexer) FromArgs(args ...any) ([]byte, error) { @@ -86,5 +92,5 @@ func (emi *LabelIndexer) FromArgs(args ...any) ([]byte, error) { labelValues = append(labelValues, value) } - return []byte(strings.Join(labelValues, "/")), nil + return emi.genKey(labelValues), nil } diff --git a/test/e2e/apisix/route.go b/test/e2e/apisix/route.go index 92cf9b7f3..eb447f63d 100644 --- a/test/e2e/apisix/route.go +++ b/test/e2e/apisix/route.go @@ -374,6 +374,42 @@ spec: applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin-service-e2e-test"}, new(apiv2.ApisixUpstream), apisixUpstreamSpec1) Eventually(request).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) }) + + It("Multiple ApisixRoute with same prefix name", func() { + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + hosts: + - %s + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + By("apply ApisixRoute") + var apisixRoute apiv2.ApisixRoute + for _, id := range []string{"11111", "1111", "111", "11", "1"} { + name := fmt.Sprintf("route-%s", id) + host := fmt.Sprintf("httpbin-%s", id) + applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: name}, &apisixRoute, fmt.Sprintf(apisixRouteSpec, name, host)) + } + + By("verify ApisixRoute works") + for _, id := range []string{"1", "11", "111", "1111", "11111"} { + host := fmt.Sprintf("httpbin-%s", id) + Eventually(func() int { + return s.NewAPISIXClient().GET("/get").WithHost(host).Expect().Raw().StatusCode + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + } + }) }) Context("Test ApisixRoute reference ApisixUpstream", func() { diff --git a/test/e2e/apisix/status.go b/test/e2e/apisix/status.go index 1c72e4128..7bc0b6a53 100644 --- a/test/e2e/apisix/status.go +++ b/test/e2e/apisix/status.go @@ -39,10 +39,6 @@ var _ = Describe("Test CRD Status", Label("apisix.apache.org", "v2", "apisixrout applier = framework.NewApplier(s.GinkgoT, s.K8sClient, s.CreateResourceFromString) ) - assertion := func(actualOrCtx any, args ...any) AsyncAssertion { - return Eventually(actualOrCtx).WithArguments(args...).WithTimeout(30 * time.Second).ProbeEvery(time.Second) - } - Context("Test ApisixRoute Sync Status", func() { BeforeEach(func() { By("create GatewayProxy") @@ -95,22 +91,16 @@ spec: - name: non-existent-plugin enable: true ` - - getRequest := func(path string) func() int { - return func() int { - return s.NewAPISIXClient().GET(path).WithHost("httpbin").Expect().Raw().StatusCode - } - } - It("unknown plugin", func() { if os.Getenv("PROVIDER_TYPE") == "apisix-standalone" { Skip("apisix standalone does not validate unknown plugins") } By("apply ApisixRoute with valid plugin") - applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, arWithInvalidPlugin) + err := s.CreateResourceFromString(arWithInvalidPlugin) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute with valid plugin") By("check ApisixRoute status") - assertion(func() string { + s.RetryAssertion(func() string { output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") return output }).Should( @@ -124,67 +114,62 @@ spec: By("Update ApisixRoute") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, ar) - By("check ApisixRoute status") - assertion(func() string { - output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") - return output - }).Should( - And( - ContainSubstring(`status: "True"`), - ContainSubstring(`reason: Accepted`), - ), - ) - By("check route in APISIX") - assertion(getRequest("/get")).Should(Equal(200), "should be able to access the route") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(200), + }) }) It("dataplane unavailable", func() { By("apply ApisixRoute") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, ar) - By("check ApisixRoute status") - assertion(func() string { - output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") - return output - }).Should( - And( - ContainSubstring(`status: "True"`), - ContainSubstring(`reason: Accepted`), - ), - ) - By("check route in APISIX") - assertion(getRequest("/get")).Should(Equal(200), "should be able to access the route") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Headers: map[string]string{"Host": "httpbin"}, + Check: scaffold.WithExpectedStatus(200), + }) s.Deployer.ScaleDataplane(0) By("check ApisixRoute status") - assertion(func() string { + s.RetryAssertion(func() string { output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") return output - }).Should( - And( - ContainSubstring(`status: "False"`), - ContainSubstring(`reason: SyncFailed`), - ), - ) + }).WithTimeout(80 * time.Second). + Should( + And( + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ), + ) s.Deployer.ScaleDataplane(1) By("check ApisixRoute status after scaling up") - assertion(func() string { + s.RetryAssertion(func() string { output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") return output - }).Should( - And( - ContainSubstring(`status: "True"`), - ContainSubstring(`reason: Accepted`), - ), - ) + }).WithTimeout(80 * time.Second). + Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring(`reason: Accepted`), + ), + ) By("check route in APISIX") - assertion(getRequest("/get")).Should(Equal(200), "should be able to access the route") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(200), + }) }) }) @@ -276,67 +261,55 @@ spec: AfterEach(func() { _ = s.DeleteResource("Gateway", "apisix") }) - getRequest := func(path string) func() int { - return func() int { - return s.NewAPISIXClient().GET(path).WithHost("httpbin").Expect().Raw().StatusCode - } - } - var resourceApplied = func(resourType, resourceName, resourceRaw string, observedGeneration int) { - Expect(s.CreateResourceFromString(resourceRaw)). - NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourType)) - - Eventually(func() string { - hryaml, err := s.GetResourceYaml(resourType, resourceName) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting %s yaml", resourType)) - return hryaml - }, "8s", "2s"). - Should( - SatisfyAll( - ContainSubstring(`status: "True"`), - ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)), - ), - fmt.Sprintf("checking %s condition status", resourType), - ) - time.Sleep(5 * time.Second) - } It("dataplane unavailable", func() { By("Create HTTPRoute") - resourceApplied("HTTPRoute", "httpbin", httproute, 1) + err := s.CreateResourceFromString(httproute) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") By("check route in APISIX") - assertion(getRequest("/get")).Should(Equal(200), "should be able to access the route") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(200), + }) s.Deployer.ScaleDataplane(0) - time.Sleep(10 * time.Second) By("check ApisixRoute status") - assertion(func() string { + s.RetryAssertion(func() string { output, _ := s.GetOutputFromString("httproute", "httpbin", "-o", "yaml") return output - }).Should( - And( - ContainSubstring(`status: "False"`), - ContainSubstring(`reason: SyncFailed`), - ), - ) + }).WithTimeout(80 * time.Second). + Should( + And( + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ), + ) s.Deployer.ScaleDataplane(1) - time.Sleep(10 * time.Second) By("check ApisixRoute status after scaling up") - assertion(func() string { + s.RetryAssertion(func() string { output, _ := s.GetOutputFromString("httproute", "httpbin", "-o", "yaml") return output - }).Should( - And( - ContainSubstring(`status: "True"`), - ContainSubstring(`reason: Accepted`), - ), - ) + }).WithTimeout(80 * time.Second). + Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring(`reason: Accepted`), + ), + ) By("check route in APISIX") - assertion(getRequest("/get")).Should(Equal(200), "should be able to access the route") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(200), + }) }) }) }) diff --git a/test/e2e/crds/backendtrafficpolicy.go b/test/e2e/crds/backendtrafficpolicy.go index 4dc2ca922..82b88a201 100644 --- a/test/e2e/crds/backendtrafficpolicy.go +++ b/test/e2e/crds/backendtrafficpolicy.go @@ -19,7 +19,6 @@ package gatewayapi import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -130,33 +129,55 @@ spec: }) It("should rewrite upstream host", func() { s.ResourceApplied("BackendTrafficPolicy", "httpbin", createUpstreamHost, 1) - s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.org"). - Expect(). - Status(200). - Body().Contains("httpbin.example.com") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.org", + Headers: map[string]string{ + "Host": "httpbin.org", + }, + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedBodyContains( + "httpbin.example.com", + ), + }, + }) s.ResourceApplied("BackendTrafficPolicy", "httpbin", updateUpstreamHost, 2) - s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.org"). - Expect(). - Status(200). - Body().Contains("httpbin.update.example.com") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.org", + Headers: map[string]string{ + "Host": "httpbin.org", + }, + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedBodyContains( + "httpbin.update.example.com", + ), + }, + }) err := s.DeleteResourceFromString(createUpstreamHost) Expect(err).NotTo(HaveOccurred(), "deleting BackendTrafficPolicy") - time.Sleep(5 * time.Second) - s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.org"). - Expect(). - Status(200). - Body(). - NotContains("httpbin.update.example.com"). - NotContains("httpbin.example.com") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.org", + Headers: map[string]string{ + "Host": "httpbin.org", + }, + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedBodyNotContains( + "httpbin.update.example.com", + "httpbin.example.com", + ), + }, + }) }) }) }) @@ -231,7 +252,6 @@ spec: By("create Ingress with GatewayProxy IngressClass") err = s.CreateResourceFromString(defaultIngress) Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") - time.Sleep(5 * time.Second) } Context("Rewrite Upstream Host", func() { @@ -265,34 +285,36 @@ spec: BeforeEach(beforeEach) It("should rewrite upstream host", func() { + reqAssert := &scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.org", + Headers: map[string]string{ + "Host": "httpbin.org", + }, + } s.ResourceApplied("BackendTrafficPolicy", "httpbin", createUpstreamHost, 1) - s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.org"). - Expect(). - Status(200). - Body().Contains("httpbin.example.com") + s.RequestAssert(reqAssert.SetChecks( + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedBodyContains("httpbin.example.com"), + )) s.ResourceApplied("BackendTrafficPolicy", "httpbin", updateUpstreamHost, 2) - s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.org"). - Expect(). - Status(200). - Body().Contains("httpbin.update.example.com") + s.RequestAssert(reqAssert.SetChecks( + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedBodyContains("httpbin.update.example.com"), + )) err := s.DeleteResourceFromString(createUpstreamHost) Expect(err).NotTo(HaveOccurred(), "deleting BackendTrafficPolicy") - time.Sleep(5 * time.Second) - s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.org"). - Expect(). - Status(200). - Body(). - NotContains("httpbin.update.example.com"). - NotContains("httpbin.example.com") + s.RequestAssert(reqAssert.SetChecks( + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedBodyNotContains( + "httpbin.update.example.com", + "httpbin.example.com", + ), + )) }) }) }) diff --git a/test/e2e/crds/consumer.go b/test/e2e/crds/consumer.go index 879d60489..26b8554f5 100644 --- a/test/e2e/crds/consumer.go +++ b/test/e2e/crds/consumer.go @@ -19,8 +19,6 @@ package gatewayapi import ( "fmt" - "net/http" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -161,35 +159,47 @@ spec: s.ResourceApplied("Consumer", "consumer-sample", limitCountConsumer, 1) s.ResourceApplied("Consumer", "consumer-sample2", unlimitConsumer, 1) - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(200), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(200), + }) By("trigger limit-count") - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(503) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(503), + }) for i := 0; i < 10; i++ { - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key2"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key2", + }, + Check: scaffold.WithExpectedStatus(200), + }) } }) }) @@ -244,72 +254,98 @@ spec: It("Create/Update/Delete", func() { s.ResourceApplied("Consumer", "consumer-sample", defaultCredential, 1) - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key2"). - WithHost("httpbin.org"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(200), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key2", + }, + Check: scaffold.WithExpectedStatus(200), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(200), + }) By("update Consumer") s.ResourceApplied("Consumer", "consumer-sample", updateCredential, 2) - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(401) - - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key2"). - WithHost("httpbin.org"). - Expect(). - Status(401) - - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "consumer-key"). - WithHost("httpbin.org"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(401), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key2", + }, + Check: scaffold.WithExpectedStatus(401), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "consumer-key", + }, + Check: scaffold.WithExpectedStatus(200), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(200), + }) By("delete Consumer") err := s.DeleteResourceFromString(updateCredential) Expect(err).NotTo(HaveOccurred(), "deleting Consumer") - time.Sleep(5 * time.Second) - - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(401) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(401), + }) }) - }) + }) Context("SecretRef", func() { var keyAuthSecret = ` apiVersion: v1 @@ -370,61 +406,79 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating basic-auth secret") s.ResourceApplied("Consumer", "consumer-sample", defaultConsumer, 1) - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(200), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(200), + }) // update basic-auth password err = s.CreateResourceFromString(basicAuthSecret2) Expect(err).NotTo(HaveOccurred(), "creating basic-auth secret") // use the old password will get 401 - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Raw().StatusCode - }).WithTimeout(8 * time.Second).ProbeEvery(time.Second). - Should(Equal(http.StatusUnauthorized)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(401), + }) // use the new password will get 200 - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password-new"). - WithHost("httpbin.org"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password-new", + }, + Check: scaffold.WithExpectedStatus(200), + }) By("delete consumer") err = s.DeleteResourceFromString(defaultConsumer) Expect(err).NotTo(HaveOccurred(), "deleting consumer") - time.Sleep(5 * time.Second) - - s.NewAPISIXClient(). - GET("/get"). - WithHeader("apikey", "sample-key"). - WithHost("httpbin.org"). - Expect(). - Status(401) - - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(401) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + Headers: map[string]string{ + "apikey": "sample-key", + }, + Check: scaffold.WithExpectedStatus(401), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(401), + }) }) }) @@ -471,12 +525,16 @@ spec: s.ResourceApplied("Consumer", "consumer-sample", defaultCredential, 1) // verify basic-auth works - s.NewAPISIXClient(). - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(200), + }) By("create additional gateway group to get new admin key") var err error @@ -490,26 +548,35 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating APISIX client for additional gateway group") By("Consumer not found for additional gateway group") - client. - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(404), + }) By("update GatewayProxy with new admin key") updatedProxy := fmt.Sprintf(updatedGatewayProxy, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey) err = s.CreateResourceFromString(updatedProxy) Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy") - time.Sleep(5 * time.Second) By("verify Consumer works for additional gateway group") - client. - GET("/get"). - WithBasicAuth("sample-user", "sample-password"). - WithHost("httpbin.org"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: "GET", + Path: "/get", + Host: "httpbin.org", + BasicAuth: &scaffold.BasicAuth{ + Username: "sample-user", + Password: "sample-password", + }, + Check: scaffold.WithExpectedStatus(200), + }) }) }) }) diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index affa4bfb5..6b4adbbc8 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -93,6 +93,13 @@ spec: volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 9080 volumes: - name: config-source configMap: diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index 37504809b..f6dc990aa 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -68,7 +68,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: apisix-ingress-manager-role + name: {{ .Namespace }}-apisix-ingress-manager-role rules: - apiGroups: - "" @@ -244,7 +244,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: apisix-ingress-metrics-auth-role + name: {{ .Namespace }}-apisix-ingress-metrics-auth-role rules: - apiGroups: - authentication.k8s.io @@ -262,7 +262,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: apisix-ingress-metrics-reader + name: {{ .Namespace }}-apisix-ingress-metrics-reader rules: - nonResourceURLs: - /metrics @@ -280,7 +280,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: apisix-ingress-leader-election-role + name: {{ .Namespace }}-apisix-ingress-leader-election-role subjects: - kind: ServiceAccount name: apisix-ingress-controller-manager @@ -292,11 +292,11 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: apisix-ingress - name: apisix-ingress-manager-rolebinding + name: {{ .Namespace }}-apisix-ingress-manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: apisix-ingress-manager-role + name: {{ .Namespace }}-apisix-ingress-manager-role subjects: - kind: ServiceAccount name: apisix-ingress-controller-manager @@ -305,11 +305,11 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: apisix-ingress-metrics-auth-rolebinding + name: {{ .Namespace }}-apisix-ingress-metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: apisix-ingress-metrics-auth-role + name: {{ .Namespace }}-apisix-ingress-metrics-auth-role subjects: - kind: ServiceAccount name: apisix-ingress-controller-manager @@ -320,14 +320,13 @@ apiVersion: v1 kind: ConfigMap metadata: name: ingress-config + namespace: {{ .Namespace }} data: config.yaml: | log_level: "debug" - controller_name: {{ .ControllerName | default "apisix.apache.org/apisix-ingress-controller" }} - leader_election_id: "apisix-ingress-controller-leader" - + exec_adc_timeout: 5s provider: type: {{ .ProviderType | default "apisix-standalone" }} sync_period: {{ .ProviderSyncPeriod | default "0s" }} diff --git a/test/e2e/gatewayapi/gatewayproxy.go b/test/e2e/gatewayapi/gatewayproxy.go index dfcbba9be..f8fc56f77 100644 --- a/test/e2e/gatewayapi/gatewayproxy.go +++ b/test/e2e/gatewayapi/gatewayproxy.go @@ -123,25 +123,6 @@ spec: port: 80 ` - var resourceApplied = func(resourceType, resourceName, resourceRaw string, observedGeneration int) { - Expect(s.CreateResourceFromString(resourceRaw)). - NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourceType)) - - Eventually(func() string { - hryaml, err := s.GetResourceYaml(resourceType, resourceName) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting %s yaml", resourceType)) - return hryaml - }).WithTimeout(8*time.Second).ProbeEvery(2*time.Second). - Should( - SatisfyAll( - ContainSubstring(`status: "True"`), - ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)), - ), - fmt.Sprintf("checking %s condition status", resourceType), - ) - time.Sleep(3 * time.Second) - } - var ( gatewayClassName string ) @@ -176,43 +157,40 @@ spec: Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") }) - AfterEach(func() { - By("Clean up resources") - _ = s.DeleteResourceFromString(fmt.Sprintf(httpRouteForTest, "apisix")) - _ = s.DeleteResourceFromString(fmt.Sprintf(gatewayWithProxy, gatewayClassName)) - _ = s.DeleteResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, s.Deployer.GetAdminEndpoint(), s.AdminKey())) - }) - Context("Test Gateway with enabled GatewayProxy plugin", func() { It("Should apply plugin configuration when enabled", func() { By("Create HTTPRoute for Gateway with GatewayProxy") - resourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "apisix"), 1) + s.ResourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "apisix"), 1) By("Check if the plugin is applied") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("example.com"). - Expect(). - Status(200) - - resp.Header("X-Proxy-Test").IsEqual("enabled") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedHeader("X-Proxy-Test", "enabled"), + }, + }) By("Update GatewayProxy with disabled plugin") err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithDisabledPlugin, s.Deployer.GetAdminEndpoint(), s.AdminKey())) Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy with disabled plugin") - time.Sleep(5 * time.Second) By("Create HTTPRoute for Gateway with GatewayProxy") - resourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "apisix"), 1) + s.ResourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "apisix"), 1) By("Check if the plugin is not applied") - resp = s.NewAPISIXClient(). - GET("/get"). - WithHost("example.com"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedHeader("X-Proxy-Test", ""), + }, + }) - resp.Header("X-Proxy-Test").IsEmpty() }) }) @@ -238,20 +216,19 @@ spec: By("Update GatewayProxy with invalid endpoint") err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithInvalidEndpoint, s.Deployer.GetAdminEndpoint(), s.AdminKey())) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with enabled plugin") - time.Sleep(5 * time.Second) By("Create HTTPRoute") - resourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "apisix"), 1) - - expectRequest := func() bool { - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("example.com"). - Expect().Raw() - return resp.StatusCode == 200 && resp.Header.Get("X-Proxy-Test") == "" - } - - Eventually(expectRequest).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(BeTrue()) + s.ResourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "apisix"), 1) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "example.com", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(200), + scaffold.WithExpectedHeader("X-Proxy-Test", ""), + }, + }) }) }) @@ -311,12 +288,10 @@ spec: err := s.CreateResourceFromString(gatewayProxyWithValidProvider) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with valid provider") - Eventually(func() string { - gpYaml, err := s.GetResourceYaml("GatewayProxy", "apisix-proxy-config") - Expect(err).NotTo(HaveOccurred(), "getting GatewayProxy yaml") + s.RetryAssertion(func() string { + gpYaml, _ := s.GetResourceYaml("GatewayProxy", "apisix-proxy-config") return gpYaml - }).WithTimeout(8*time.Second).ProbeEvery(2*time.Second). - Should(ContainSubstring(`"type":"ControlPlane"`), "checking GatewayProxy is applied") + }).Should(ContainSubstring(`"type":"ControlPlane"`), "checking GatewayProxy is applied") }) }) }) diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index 01e30ee89..11e990a96 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -24,7 +24,6 @@ import ( "strings" "time" - "github.com/gruntwork-io/terratest/modules/retry" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" @@ -131,86 +130,80 @@ spec: name: apisix-proxy-config ` - var ResourceApplied = func(resourType, resourceName, resourceRaw string, observedGeneration int) { - Expect(s.CreateResourceFromString(resourceRaw)). - NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourType)) - - Eventually(func() string { - hryaml, err := s.GetResourceYaml(resourType, resourceName) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting %s yaml", resourType)) - return hryaml - }, "8s", "2s"). - Should( - SatisfyAll( - ContainSubstring(`status: "True"`), - ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)), - ), - fmt.Sprintf("checking %s condition status", resourType), - ) - time.Sleep(5 * time.Second) - } - var beforeEachHTTP = func() { - By("create GatewayProxy") - err := s.CreateResourceFromString(getGatewayProxySpec()) - Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) - - By("create GatewayClass") - gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") - Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) - - By("check GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") - - By("create Gateway") - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.Namespace()) - Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) - - By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", "apisix") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + Expect(s.CreateResourceFromString(getGatewayProxySpec())). + NotTo(HaveOccurred(), "creating GatewayProxy") + + gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Nanosecond()) + Expect(s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "")). + NotTo(HaveOccurred(), "creating GatewayClass") + + s.RetryAssertion(func() string { + gcyaml, _ := s.GetResourceYaml("GatewayClass", gatewayClassName) + return gcyaml + }).Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), + "check GatewayClass condition", + ) + + Expect(s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGateway, gatewayClassName), s.Namespace())). + NotTo(HaveOccurred(), "creating Gateway") + + s.RetryAssertion(func() string { + gcyaml, _ := s.GetResourceYaml("Gateway", "apisix") + return gcyaml + }).Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controlle"), + ), + "check Gateway condition status", + ) } var beforeEachHTTPS = func() { By("create GatewayProxy") err := s.CreateResourceFromString(getGatewayProxySpec()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) secretName := _secretName createSecret(s, secretName) - By("create GatewayClass") - gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") - Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) - By("check GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking GatewayClass condition message") + By("create GatewayClass") + gatewayClassName := fmt.Sprintf("apisix-%d", time.Now().Nanosecond()) + Expect(s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "")). + NotTo(HaveOccurred(), "creating GatewayClass") + + s.RetryAssertion(func() string { + gcyaml, _ := s.GetResourceYaml("GatewayClass", gatewayClassName) + return gcyaml + }).Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), + "check GatewayClass condition", + ) By("create Gateway") err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName), s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) - By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", "apisix") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controller"), "checking Gateway condition message") + s.RetryAssertion(func() string { + gcyaml, _ := s.GetResourceYaml("Gateway", "apisix") + return gcyaml + }).Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gateway has been accepted by the apisix-ingress-controlle"), + ), + "check Gateway condition status", + ) } + Context("HTTPRoute with HTTPS Gateway", func() { var exactRouteByGet = ` apiVersion: gateway.networking.k8s.io/v1 @@ -236,24 +229,26 @@ spec: It("Create/Updtea/Delete HTTPRoute", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) By("access dataplane to check the HTTPRoute") - s.NewAPISIXHttpsClient("api6.com"). - GET("/get"). - WithHost("api6.com"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "api6.com", + Check: scaffold.WithExpectedStatus(200), + }) + By("delete HTTPRoute") err := s.DeleteResourceFromString(exactRouteByGet) Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute") - time.Sleep(5 * time.Second) - s.NewAPISIXHttpsClient("api6.com"). - GET("/get"). - WithHost("api6.com"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "api6.com", + Check: scaffold.WithExpectedStatus(404), + }) }) }) @@ -342,12 +337,17 @@ spec: additionalGatewayClassName = fmt.Sprintf("apisix-%d", time.Now().Unix()) err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, additionalGatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating additional GatewayClass") - time.Sleep(5 * time.Second) + By("Check additional GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", additionalGatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting additional GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking additional GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), "checking additional GatewayClass condition message") + s.RetryAssertion(func() string { + gcyaml, _ := s.GetResourceYaml("GatewayClass", additionalGatewayClassName) + return gcyaml + }).Should( + And( + ContainSubstring(`status: "True"`), + ContainSubstring("message: the gatewayclass has been accepted by the apisix-ingress-controller"), + ), + ) additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey) err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, resources.DataplaneService.Namespace) @@ -359,52 +359,56 @@ spec: additionalSvc.Namespace, ) Expect(err).NotTo(HaveOccurred(), "creating additional Gateway") - time.Sleep(5 * time.Second) }) It("HTTPRoute should be accessible through both gateways", func() { By("Create HTTPRoute referencing both gateways") multiGatewayRoute := fmt.Sprintf(multiGatewayHTTPRoute, s.Namespace(), additionalSvc.Namespace) - ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1) + s.ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1) By("Access through default gateway") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("Access through additional gateway") client, err := s.NewAPISIXClientForGateway(additionalGatewayGroupID) Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") - client. - GET("/get"). - WithHost("httpbin-additional.example"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: "GET", + Path: "/get", + Host: "httpbin-additional.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("Delete Additional Gateway") err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalSvc.Namespace) Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway") - time.Sleep(5 * time.Second) By("HTTPRoute should still be accessible through default gateway") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("HTTPRoute should not be accessible through additional gateway") client, err = s.NewAPISIXClientForGateway(additionalGatewayGroupID) Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") - client. - GET("/get"). - WithHost("httpbin-additional.example"). - Expect(). - Status(http.StatusNotFound) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: "GET", + Path: "/get", + Host: "httpbin-additional.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) }) @@ -416,7 +420,7 @@ metadata: name: httpbin-external-domain spec: type: ExternalName - externalName: httpbin.org + externalName: httpbin-service-e2e-test --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute @@ -520,121 +524,136 @@ spec: It("Create/Update/Delete HTTPRoute", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) By("access dataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) - By("delete HTTPRoute") - err := s.DeleteResourceFromString(exactRouteByGet) - Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute") - time.Sleep(5 * time.Second) + Expect(s.DeleteResourceFromString(exactRouteByGet)). + NotTo(HaveOccurred(), "deleting HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) It("Delete Gateway after apply HTTPRoute", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) By("access dataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - By("delete Gateway") - err := s.DeleteResource("Gateway", "apisix") - Expect(err).NotTo(HaveOccurred(), "deleting Gateway") - - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Raw().StatusCode - }).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + Expect(s.DeleteResource("Gateway", "apisix")). + NotTo(HaveOccurred(), "deleting Gateway") + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) It("Proxy External Service", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", httprouteWithExternalName, 1) + s.ResourceApplied("HTTPRoute", "httpbin", httprouteWithExternalName, 1) By("checking the external service response") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.external"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.external", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) }) It("Match Port", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", invalidBackendPort, 1) + s.ResourceApplied("HTTPRoute", "httpbin", invalidBackendPort, 1) - serviceResources, err := s.DefaultDataplaneResource().Service().List(context.Background()) - Expect(err).NotTo(HaveOccurred(), "listing services") - Expect(serviceResources).To(HaveLen(1), "checking service length") + s.RetryAssertion(func() error { + serviceResources, err := s.DefaultDataplaneResource().Service().List(context.Background()) + if err != nil { + return errors.Wrap(err, "listing services") + } + if len(serviceResources) != 1 { + return fmt.Errorf("expected 1 service, got %d", len(serviceResources)) + } - serviceResource := serviceResources[0] - nodes := serviceResource.Upstream.Nodes - Expect(nodes).To(HaveLen(1), "checking nodes length") - Expect(nodes[0].Port).To(Equal(80)) + serviceResource := serviceResources[0] + nodes := serviceResource.Upstream.Nodes + if len(nodes) != 1 { + return fmt.Errorf("expected 1 node, got %d", len(nodes)) + } + if nodes[0].Port != 80 { + return fmt.Errorf("expected node port 80, got %d", nodes[0].Port) + } + return nil + }).Should(Succeed(), "checking service port") }) It("Delete HTTPRoute during restart", func() { By("create HTTPRoute httpbin") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) By("create HTTPRoute httpbin2") - ResourceApplied("HTTPRoute", "httpbin2", exactRouteByGet2, 1) + s.ResourceApplied("HTTPRoute", "httpbin2", exactRouteByGet2, 1) - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin2.example"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin2.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) s.Deployer.ScaleIngress(0) - By("delete HTTPRoute httpbin2") - err := s.DeleteResource("HTTPRoute", "httpbin2") - Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute httpbin2") + Expect(s.DeleteResource("HTTPRoute", "httpbin2")). + NotTo(HaveOccurred(), "deleting HTTPRoute httpbin2") s.Deployer.ScaleIngress(1) - time.Sleep(1 * time.Minute) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin2.example"). - Expect(). - Status(404) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Timeout: 1 * time.Minute, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin2.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) }) @@ -755,92 +774,106 @@ spec: It("HTTPRoute Exact Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/get/xxx"). - WithHost("httpbin.example"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get/xxx", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) It("HTTPRoute Prefix Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", prefixRouteByStatus, 1) + s.ResourceApplied("HTTPRoute", "httpbin", prefixRouteByStatus, 1) By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/status/200"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - GET("/status/201"). - WithHost("httpbin.example"). - Expect(). - Status(201) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/status/200", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/status/201", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusCreated), + }) }) It("HTTPRoute Method Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", methodRouteGETAndDELETEByAnything, 1) + s.ResourceApplied("HTTPRoute", "httpbin", methodRouteGETAndDELETEByAnything, 1) By("access daataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/anything"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - DELETE("/anything"). - WithHost("httpbin.example"). - Expect(). - Status(200) - - s.NewAPISIXClient(). - POST("/anything"). - WithHost("httpbin.example"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/anything", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "DELETE", + Path: "/anything", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "POST", + Path: "/anything", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) It("HTTPRoute Vars Match", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", varsRoute, 1) + s.ResourceApplied("HTTPRoute", "httpbin", varsRoute, 1) By("access dataplane to check the HTTPRoute") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(http.StatusNotFound) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) }) It("HTTPRoutePolicy in effect", func() { By("create HTTPRoute") s.ApplyHTTPRoute(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin"}, varsRoute) - request := func() int { - return s.NewAPISIXClient().GET("/get"). - WithHost("httpbin.example").WithHeader("X-Route-Name", "httpbin"). - Expect().Raw().StatusCode - } - Eventually(request).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("create HTTPRoutePolicy") s.ApplyHTTPRoutePolicy( @@ -850,16 +883,29 @@ spec: ) By("access dataplane to check the HTTPRoutePolicy") - Eventually(request).WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - WithHeader("X-HRP-Name", "http-route-policy-0"). - WithQuery("hrp_name", "http-route-policy-0"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Query: map[string]any{ + "hrp_name": "http-route-policy-0", + }, + Headers: map[string]string{ + "X-Route-Name": "httpbin", + "X-HRP-Name": "http-route-policy-0", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("update HTTPRoutePolicy") const changedHTTPRoutePolicy = ` @@ -886,24 +932,31 @@ spec: ) // use the old vars cannot match any route - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - WithHeader("X-HRP-Name", "http-route-policy-0"). - WithQuery("hrp_name", "http-route-policy-0"). - Expect().Raw().StatusCode - }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Query: map[string]any{ + "hrp_name": "http-route-policy-0", + }, + Headers: map[string]string{ + "X-Route-Name": "httpbin", + "X-HRP-Name": "http-route-policy-0", + }, + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) // use the new vars can match the route - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - WithHeader("X-HRP-Name", "new-hrp-name"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + "X-HRP-Name": "new-hrp-name", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("delete the HTTPRoutePolicy") err := s.DeleteResource("HTTPRoutePolicy", "http-route-policy-0") @@ -913,18 +966,15 @@ spec: return err.Error() }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(ContainSubstring(`httproutepolicies.apisix.apache.org "http-route-policy-0" not found`)) // access the route without additional vars should be OK - message := retry.DoWithRetry(s.GinkgoT, "", 10, time.Second, func() (string, error) { - statusCode := s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect().Raw().StatusCode - if statusCode != http.StatusOK { - return "", errors.Errorf("unexpected status code: %v", statusCode) - } - return "request OK", nil + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), }) - s.Logf(message) }) It("HTTPRoutePolicy conflicts", func() { @@ -1026,13 +1076,15 @@ spec: } // assert that conflict policies are not in effect - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect().Raw().StatusCode - }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("delete HTTPRoutePolicies") err := s.DeleteResource("HTTPRoutePolicy", "http-route-policy-2") @@ -1048,13 +1100,15 @@ spec: }, ) } - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect().Raw().StatusCode - }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) By("update HTTPRoutePolicy") err = s.CreateResourceFromString(httpRoutePolicy1Priority20) @@ -1077,13 +1131,16 @@ spec: }, ) } - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect().Raw().StatusCode - }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) }) It("HTTPRoutePolicy status changes on HTTPRoute deleting", func() { @@ -1098,42 +1155,49 @@ spec: ) By("access dataplane to check the HTTPRoutePolicy") - Eventually(func() int { - return s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - Expect().Raw().StatusCode - }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) - - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - WithHeader("X-HRP-Name", "http-route-policy-0"). - WithQuery("hrp_name", "http-route-policy-0"). - Expect(). - Status(http.StatusOK) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Route-Name": "httpbin", + }, + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Query: map[string]any{ + "hrp_name": "http-route-policy-0", + }, + Headers: map[string]string{ + "X-Route-Name": "httpbin", + "X-HRP-Name": "http-route-policy-0", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("delete the HTTPRoute, assert the HTTPRoutePolicy's status will be changed") - err := s.DeleteResource("HTTPRoute", "httpbin") - Expect(err).NotTo(HaveOccurred(), "deleting HTTPRoute") - message := retry.DoWithRetry(s.GinkgoT, "request the deleted route", 10, time.Second, func() (string, error) { - statusCode := s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - WithHeader("X-Route-Name", "httpbin"). - WithHeader("X-HRP-Name", "http-route-policy-0"). - WithQuery("hrp_name", "http-route-policy-0"). - Expect().Raw().StatusCode - if statusCode != http.StatusNotFound { - return "", errors.Errorf("unexpected status code: %v", statusCode) - } - return "the route is deleted", nil + Expect(s.DeleteResource("HTTPRoute", "httpbin")). + NotTo(HaveOccurred(), "deleting HTTPRoute") + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Query: map[string]any{ + "hrp_name": "http-route-policy-0", + }, + Headers: map[string]string{ + "X-Route-Name": "httpbin", + "X-HRP-Name": "http-route-policy-0", + }, + Check: scaffold.WithExpectedStatus(http.StatusNotFound), }) - s.Logf(message) - err = framework.PollUntilHTTPRoutePolicyHaveStatus(s.K8sClient, 8*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "http-route-policy-0"}, + err := framework.PollUntilHTTPRoutePolicyHaveStatus(s.K8sClient, 8*time.Second, types.NamespacedName{Namespace: s.Namespace(), Name: "http-route-policy-0"}, func(hrp *v1alpha1.HTTPRoutePolicy) bool { return len(hrp.Status.Ancestors) == 0 }, @@ -1354,63 +1418,73 @@ spec: It("HTTPRoute RequestHeaderModifier", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", reqHeaderModifyByHeaders, 1) + s.ResourceApplied("HTTPRoute", "httpbin", reqHeaderModifyByHeaders, 1) By("access daataplane to check the HTTPRoute") - respExp := s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.example"). - WithHeader("X-Req-Add", "test"). - WithHeader("X-Req-Removed", "test"). - WithHeader("X-Req-Set", "test"). - Expect() - - respExp.Status(200) - respExp.Body(). - Contains(`"X-Req-Add": "test,add"`). - Contains(`"X-Req-Set": "set"`). - NotContains(`"X-Req-Removed": "remove"`) - + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.example", + Headers: map[string]string{ + "X-Req-Add": "test", + "X-Req-Removed": "test", + "X-Req-Set": "test", + }, + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedBodyContains(`"X-Req-Add": "test,add"`, `"X-Req-Set": "set"`), + scaffold.WithExpectedBodyNotContains(`"X-Req-Removed": "remove"`), + }, + }) }) It("HTTPRoute ResponseHeaderModifier", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", respHeaderModifyByHeaders, 1) + s.ResourceApplied("HTTPRoute", "httpbin", respHeaderModifyByHeaders, 1) By("access daataplane to check the HTTPRoute") - respExp := s.NewAPISIXClient(). - GET("/headers"). - WithHost("httpbin.example"). - Expect() - - respExp.Status(200) - respExp.Header("X-Resp-Add").IsEqual("add") - respExp.Header("X-Resp-Set").IsEqual("set") - respExp.Header("Server").IsEmpty() - respExp.Body(). - NotContains(`"X-Resp-Add": "add"`). - NotContains(`"X-Resp-Set": "set"`). - NotContains(`"Server"`) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedHeaders(map[string]string{ + "X-Resp-Add": "add", + "X-Resp-Set": "set", + "Server": "", + }), + scaffold.WithExpectedBodyNotContains(`"X-Resp-Add": "add"`, `"X-Resp-Set": "set"`, `"Server"`), + }, + }) }) It("HTTPRoute RequestRedirect", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", httpsRedirectByHeaders, 1) - - s.NewAPISIXClient().GET("/headers"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusFound). - Header("Location").IsEqual("https://httpbin.example:9443/headers") + s.ResourceApplied("HTTPRoute", "httpbin", httpsRedirectByHeaders, 1) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusFound), + scaffold.WithExpectedHeader("Location", "https://httpbin.example:9443/headers"), + }, + }) By("update HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", hostnameRedirectByHeaders, 2) - - s.NewAPISIXClient().GET("/headers"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusMovedPermanently). - Header("Location").IsEqual("http://httpbin.org/headers") + s.ResourceApplied("HTTPRoute", "httpbin", hostnameRedirectByHeaders, 2) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/headers", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusMovedPermanently), + scaffold.WithExpectedHeader("Location", "http://httpbin.org/headers"), + }, + }) }) It("HTTPRoute RequestMirror", func() { @@ -1472,78 +1546,89 @@ spec: - name: httpbin-service-e2e-test port: 80 ` - ResourceApplied("HTTPRoute", "httpbin", echoRoute, 1) - - time.Sleep(time.Second * 6) - - _ = s.NewAPISIXClient().GET("/headers"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusOK) + s.ResourceApplied("HTTPRoute", "httpbin", echoRoute, 1) - echoLogs := s.GetDeploymentLogs("echo") - Expect(echoLogs).To(ContainSubstring("GET /headers")) + s.RetryAssertion(func() string { + resp := s.NewAPISIXClient().GET("/headers").WithHost("httpbin.example").Expect().Raw() + if resp.StatusCode != http.StatusOK { + return fmt.Sprintf("expected status OK, got %d", resp.StatusCode) + } + return s.GetDeploymentLogs("echo") + }).WithTimeout(2 * time.Minute).Should(ContainSubstring("GET /headers")) }) It("HTTPRoute URLRewrite with ReplaceFullPath And Hostname", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", replaceFullPathAndHost, 1) + s.ResourceApplied("HTTPRoute", "httpbin", replaceFullPathAndHost, 1) By("/replace/201 should be rewritten to /headers") - s.NewAPISIXClient().GET("/replace/201"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusOK). - Body(). - Contains("replace.example.org") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/replace/201", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedBodyContains("replace.example.org"), + }, + }) By("/replace/500 should be rewritten to /headers") - s.NewAPISIXClient().GET("/replace/500"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusOK). - Body(). - Contains("replace.example.org") + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/replace/500", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusOK), + scaffold.WithExpectedBodyContains("replace.example.org"), + }, + }) }) It("HTTPRoute URLRewrite with ReplacePrefixMatch", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", replacePrefixMatch, 1) + s.ResourceApplied("HTTPRoute", "httpbin", replacePrefixMatch, 1) By("/replace/201 should be rewritten to /status/201") - s.NewAPISIXClient().GET("/replace/201"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusCreated) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/replace/201", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusCreated), + }) By("/replace/500 should be rewritten to /status/500") - s.NewAPISIXClient().GET("/replace/500"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusInternalServerError) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/replace/500", + Host: "httpbin.example", + Checks: []scaffold.ResponseCheckFunc{ + scaffold.WithExpectedStatus(http.StatusInternalServerError), + }, + }) }) It("HTTPRoute ExtensionRef", func() { By("create HTTPRoute") - err := s.CreateResourceFromString(echoPlugin) - Expect(err).NotTo(HaveOccurred(), "creating PluginConfig") - ResourceApplied("HTTPRoute", "httpbin", extensionRefEchoPlugin, 1) - - s.NewAPISIXClient().GET("/get"). - WithHeader("Host", "httpbin.example"). - Expect(). - Body(). - Contains("Hello, World!!") - - err = s.CreateResourceFromString(echoPluginUpdated) - Expect(err).NotTo(HaveOccurred(), "updating PluginConfig") - time.Sleep(5 * time.Second) + Expect(s.CreateResourceFromString(echoPlugin)). + NotTo(HaveOccurred(), "creating PluginConfig") + s.ResourceApplied("HTTPRoute", "httpbin", extensionRefEchoPlugin, 1) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedBodyContains("Hello, World!!"), + }) - s.NewAPISIXClient().GET("/get"). - WithHeader("Host", "httpbin.example"). - Expect(). - Body(). - Contains("Updated") + Expect(s.CreateResourceFromString(echoPluginUpdated)). + NotTo(HaveOccurred(), "updating PluginConfig") + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedBodyContains("Updated"), + }) }) }) @@ -1602,45 +1687,58 @@ spec: }) }) It("HTTPRoute Canary", func() { - ResourceApplied("HTTPRoute", "httpbin", sameWeiht, 1) + s.ResourceApplied("HTTPRoute", "httpbin", sameWeiht, 1) + time.Sleep(5 * time.Second) - var ( - hitNginxCnt = 0 - hitHttpbinCnt = 0 - ) - for i := 0; i < 100; i++ { - body := s.NewAPISIXClient().GET("/get"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusOK). - Body().Raw() - - if strings.Contains(body, "Hello") { - hitNginxCnt++ - } else { - hitHttpbinCnt++ + s.RetryAssertion(func() int { + var ( + hitNginxCnt = 0 + hitHttpbinCnt = 0 + ) + for i := 0; i < 20; i++ { + resp := s.NewAPISIXClient().GET("/get"). + WithHeader("Host", "httpbin.example"). + Expect() + body := resp.Body().Raw() + status := resp.Raw().StatusCode + if status != http.StatusOK { + return -100 + } + + if strings.Contains(body, "Hello") { + hitNginxCnt++ + } else { + hitHttpbinCnt++ + } } - } - Expect(hitNginxCnt - hitHttpbinCnt).To(BeNumerically("~", 0, 2)) - - ResourceApplied("HTTPRoute", "httpbin", oneWeiht, 2) - - hitNginxCnt = 0 - hitHttpbinCnt = 0 - for i := 0; i < 100; i++ { - body := s.NewAPISIXClient().GET("/get"). - WithHeader("Host", "httpbin.example"). - Expect(). - Status(http.StatusOK). - Body().Raw() - - if strings.Contains(body, "Hello") { - hitNginxCnt++ - } else { - hitHttpbinCnt++ + return hitNginxCnt - hitHttpbinCnt + }).WithTimeout(2 * time.Minute).Should(BeNumerically("~", 0, 2)) + + s.ResourceApplied("HTTPRoute", "httpbin", oneWeiht, 2) + + s.RetryAssertion(func() int { + var ( + hitNginxCnt = 0 + hitHttpbinCnt = 0 + ) + for i := 0; i < 20; i++ { + resp := s.NewAPISIXClient().GET("/get"). + WithHeader("Host", "httpbin.example"). + Expect() + body := resp.Body().Raw() + status := resp.Raw().StatusCode + if status != http.StatusOK { + return -100 + } + + if strings.Contains(body, "Hello") { + hitNginxCnt++ + } else { + hitHttpbinCnt++ + } } - } - Expect(hitHttpbinCnt - hitNginxCnt).To(Equal(100)) + return hitHttpbinCnt - hitNginxCnt + }).WithTimeout(2 * time.Minute).Should(Equal(20)) }) }) @@ -1688,14 +1786,15 @@ spec: It("Should sync HTTPRoute when GatewayProxy is updated", func() { By("create HTTPRoute") - ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) + s.ResourceApplied("HTTPRoute", "httpbin", exactRouteByGet, 1) By("verify HTTPRoute works") - s.NewAPISIXClient(). - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("create additional gateway group to get new admin key") var err error @@ -1709,24 +1808,27 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating APISIX client for additional gateway group") By("HTTPRoute not found for additional gateway group") - client. - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(404) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) By("update GatewayProxy with new admin key") updatedProxy := fmt.Sprintf(updatedGatewayProxy, s.Deployer.GetAdminEndpoint(resources.DataplaneService), resources.AdminAPIKey) err = s.CreateResourceFromString(updatedProxy) Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy") - time.Sleep(5 * time.Second) By("verify HTTPRoute works for additional gateway group") - client. - GET("/get"). - WithHost("httpbin.example"). - Expect(). - Status(200) + s.RequestAssert(&scaffold.RequestAssert{ + Client: client, + Method: "GET", + Path: "/get", + Host: "httpbin.example", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) }) }) @@ -1740,7 +1842,7 @@ metadata: name: httpbin-external-domain spec: type: ExternalName - externalName: httpbin.org + externalName: httpbin-service-e2e-test --- apiVersion: v1 kind: Service @@ -1763,7 +1865,7 @@ spec: kind: Service group: "" passHost: node - scheme: https + scheme: http --- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute @@ -1779,10 +1881,10 @@ spec: value: /headers backendRefs: - name: httpbin-external-domain - port: 443 + port: 80 weight: 1 - name: mockapi7-external-domain - port: 443 + port: 80 weight: 1 ` @@ -1797,22 +1899,14 @@ spec: totalRequests := 20 for i := 0; i < totalRequests; i++ { - resp := s.NewAPISIXClient().GET("/headers").Expect().Status(http.StatusOK) - - // Parse JSON response to get the Host header - var responseBody map[string]any - resp.JSON().Decode(&responseBody) - - if headers, ok := responseBody["headers"].(map[string]any); ok { - var host string - if host, ok = headers["Host"].(string); !ok { - host, ok = headers["host"].(string) - } - if ok && host != "" { - upstreamHosts[host]++ - } - Expect(ok).To(BeTrue(), "Host header should be present") - Expect(host).Should(Or(Equal("httpbin.org"), Equal("mock.api7.ai"))) + statusCode := s.NewAPISIXClient().GET("/headers").Expect().Raw().StatusCode + Expect(statusCode).To(Or(Equal(http.StatusOK), Equal(http.StatusMovedPermanently))) + + switch statusCode { + case http.StatusOK: + upstreamHosts["httpbin-service-e2e-test"]++ + case http.StatusMovedPermanently: + upstreamHosts["mock.api7.ai"]++ } time.Sleep(100 * time.Millisecond) // Small delay between requests } @@ -1825,16 +1919,4 @@ spec: } }) }) - - /* - Context("HTTPRoute Status Updated", func() { - }) - - Context("HTTPRoute ParentRefs With Multiple Gateway", func() { - }) - - - Context("HTTPRoute BackendRefs Discovery", func() { - }) - */ }) diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index 1d30bb575..6b537b989 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -186,7 +186,7 @@ metadata: name: httpbin-external-domain spec: type: ExternalName - externalName: httpbin.org + externalName: httpbin-service-e2e-test --- apiVersion: networking.k8s.io/v1 kind: Ingress diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index e1707e448..fb22200b2 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -66,7 +66,7 @@ func (s *APISIXDeployer) BeforeEach() { Namespace: s.namespace, } if s.opts.ControllerName == "" { - s.opts.ControllerName = fmt.Sprintf("%s/%d", DefaultControllerName, time.Now().Nanosecond()) + s.opts.ControllerName = fmt.Sprintf("%s/%s", DefaultControllerName, s.namespace) } s.finalizers = nil if s.label == nil { @@ -130,14 +130,14 @@ func (s *APISIXDeployer) AfterEach() { Expect(err).NotTo(HaveOccurred(), "cleaning up additional gateway") } - // if the test case is successful, just delete namespace - err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) - Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) - for i := len(s.finalizers) - 1; i >= 0; i-- { runWithRecover(s.finalizers[i]) } + // if the test case is successful, just delete namespace + err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) + Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) + // Wait for a while to prevent the worker node being overwhelming // (new cases will be run). time.Sleep(3 * time.Second) diff --git a/test/e2e/scaffold/assertion.go b/test/e2e/scaffold/assertion.go new file mode 100644 index 000000000..c8946f2fd --- /dev/null +++ b/test/e2e/scaffold/assertion.go @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package scaffold + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gavv/httpexpect/v2" + . "github.com/onsi/gomega" //nolint:staticcheck + "github.com/onsi/gomega/types" +) + +const ( + DefaultTimeout = 12 * time.Second + DefaultInterval = 1 * time.Second +) + +type ResponseCheckFunc func(*HTTPResponse) error + +type HTTPResponse struct { + *http.Response + + Body string +} + +type BasicAuth struct { + Username string + Password string +} + +type RequestAssert struct { + Client *httpexpect.Expect + Method string + Path string + Host string + Query map[string]any + Headers map[string]string + Body []byte + BasicAuth *BasicAuth + + Timeout time.Duration + Interval time.Duration + + Check ResponseCheckFunc + Checks []ResponseCheckFunc +} + +func (c *RequestAssert) request(method, path string, body []byte) *httpexpect.Request { + switch strings.ToUpper(method) { + case "GET": + return c.Client.GET(path) + case "POST": + return c.Client.POST(path).WithBytes(body) + case "PUT": + return c.Client.PUT(path).WithBytes(body) + case "DELETE": + return c.Client.DELETE(path) + case "PATCH": + return c.Client.PATCH(path).WithBytes(body) + default: + panic("unsupported method: " + method) + } +} + +func (c *RequestAssert) WithCheck(check ResponseCheckFunc) *RequestAssert { + c.Checks = append(c.Checks, check) + return c +} + +func (c *RequestAssert) WithChecks(checks ...ResponseCheckFunc) *RequestAssert { + c.Checks = append(c.Checks, checks...) + return c +} + +func (c *RequestAssert) SetChecks(checks ...ResponseCheckFunc) *RequestAssert { + c.Checks = checks + return c +} + +func WithExpectedStatus(status int) ResponseCheckFunc { + return func(resp *HTTPResponse) error { + if resp.StatusCode != status { + return fmt.Errorf("expected %d, but got %d", status, resp.StatusCode) + } + return nil + } +} + +func WithExpectedBodyContains(expectedBodyList ...string) ResponseCheckFunc { + return func(resp *HTTPResponse) error { + for _, body := range expectedBodyList { + if !strings.Contains(resp.Body, body) { + return fmt.Errorf("expected body to contain %q, but got %q", body, resp.Body) + } + } + return nil + } +} + +func WithExpectedBodyNotContains(unexpectedBodyList ...string) ResponseCheckFunc { + return func(resp *HTTPResponse) error { + for _, unexpectedBody := range unexpectedBodyList { + if strings.Contains(resp.Body, unexpectedBody) { + return fmt.Errorf("expected body not to contain %q, but got %q", unexpectedBody, resp.Body) + } + } + return nil + } +} + +func WithExpectedHeader(key, value string) ResponseCheckFunc { + return func(resp *HTTPResponse) error { + if resp.Header.Get(key) != value { + return fmt.Errorf("expected header %q to be %q, but got %q", + key, value, resp.Header.Get(key)) + } + return nil + } +} + +func WithExpectedHeaders(expectedHeaders map[string]string) ResponseCheckFunc { + return func(resp *HTTPResponse) error { + for key, expectedValue := range expectedHeaders { + actualValue := resp.Header.Get(key) + if actualValue != expectedValue { + return fmt.Errorf("expected header %q to be %q, but got %q", + key, expectedValue, actualValue) + } + } + return nil + } +} + +func (s *Scaffold) RequestAssert(r *RequestAssert) bool { + if r.Client == nil { + r.Client = s.NewAPISIXClient() + } + if r.Method == "" { + if len(r.Body) > 0 { + r.Method = "POST" + } else { + r.Method = "GET" + } + } + if r.Timeout == 0 { + r.Timeout = DefaultTimeout + } + if r.Interval == 0 { + r.Interval = DefaultInterval + } + if r.Check == nil && len(r.Checks) == 0 { + r.Check = WithExpectedStatus(http.StatusOK) + } else if r.Check != nil { + r.Checks = append(r.Checks, r.Check) + } + + return EventuallyWithOffset(1, func() error { + req := r.request(r.Method, r.Path, r.Body) + if len(r.Headers) > 0 { + req = req.WithHeaders(r.Headers) + } + if r.Host != "" { + req = req.WithHost(r.Host) + } + if len(r.Query) > 0 { + for key, value := range r.Query { + req = req.WithQuery(key, value) + } + } + if r.BasicAuth != nil { + req = req.WithBasicAuth(r.BasicAuth.Username, r.BasicAuth.Password) + } + expResp := req.Expect() + + resp := &HTTPResponse{ + Response: expResp.Raw(), + Body: expResp.Body().Raw(), + } + + for _, check := range r.Checks { + if err := check(resp); err != nil { + return fmt.Errorf("response check failed: %w", err) + } + } + return nil + }).WithTimeout(r.Timeout).ProbeEvery(r.Interval).Should(Succeed()) +} + +// RetryAssertion provides a reusable Eventually-based assertion +type RetryAssertion struct { + timeout time.Duration + interval time.Duration + + args []any + actualOrCtx any +} + +// NewRetryAssertion creates a RetryAssertion with defaults +func (s *Scaffold) RetryAssertion(actualOrCtx any, args ...any) *RetryAssertion { + return &RetryAssertion{ + timeout: DefaultTimeout, + interval: DefaultInterval, + args: args, + actualOrCtx: actualOrCtx, + } +} + +// WithTimeout sets the timeout +func (r *RetryAssertion) WithTimeout(timeout time.Duration) *RetryAssertion { + r.timeout = timeout + return r +} + +// WithInterval sets the polling interval +func (r *RetryAssertion) WithInterval(interval time.Duration) *RetryAssertion { + r.interval = interval + return r +} + +// Should runs the Eventually assertion with the given matcher +func (r *RetryAssertion) Should(matcher types.GomegaMatcher, optionalDescription ...any) bool { + return EventuallyWithOffset(1, r.actualOrCtx, r.args...). + WithTimeout(r.timeout). + ProbeEvery(r.interval). + Should(matcher, optionalDescription...) +} + +// ShouldNot runs the Eventually assertion with the given matcher +func (r *RetryAssertion) ShouldNot(matcher types.GomegaMatcher, optionalDescription ...any) bool { + return EventuallyWithOffset(1, r.actualOrCtx, r.args...). + WithTimeout(r.timeout). + ProbeEvery(r.interval). + ShouldNot(matcher, optionalDescription...) +} diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index 890552c7a..0612d4c09 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -184,19 +184,17 @@ func (s *Scaffold) ResourceApplied(resourType, resourceName, resourceRaw string, Expect(s.CreateResourceFromString(resourceRaw)). NotTo(HaveOccurred(), fmt.Sprintf("creating %s", resourType)) - Eventually(func() string { + s.RetryAssertion(func() string { hryaml, err := s.GetResourceYaml(resourType, resourceName) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("getting %s yaml", resourType)) return hryaml - }).WithTimeout(8*time.Second).ProbeEvery(2*time.Second). - Should( - SatisfyAll( - ContainSubstring(`status: "True"`), - ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)), - ), - fmt.Sprintf("checking %s condition status", resourType), - ) - time.Sleep(3 * time.Second) + }).Should( + SatisfyAll( + ContainSubstring(`status: "True"`), + ContainSubstring(fmt.Sprintf("observedGeneration: %d", observedGeneration)), + ), + fmt.Sprintf("checking %s condition status", resourType), + ) } func (s *Scaffold) ApplyDefaultGatewayResource(