(backport) chore: add more conformance-test report for gateway-api (#2557) (#263)

Author: AlinsRan

.github/workflows/apisix-conformance-test.yml

@@ -34,6 +34,8 @@ permissions:
 
 jobs:
   conformance-test:
+    env:
+      CONFORMANCE_TEST_REPORT_OUTPUT: /tmp/apisix-ingress-controller-conformance-report.yaml
     timeout-minutes: 60
     strategy:
       matrix:
@@ -62,6 +64,10 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Install kind
+        run: |
+          go install sigs.k8s.io/[email protected]
+
       - name: Build images
         env:
           TAG: dev
@@ -97,27 +103,31 @@ jobs:
         env:
           PROVIDER_TYPE: ${{ matrix.provider_type }}
         run: |
-          make conformance-test-standalone
+          make conformance-test
       
-      - name: Get Logs from api7-ingress-controller
+      - name: Show Conformance Report
+        shell: bash
+        run: |
+          cat ${CONFORMANCE_TEST_REPORT_OUTPUT}
+
+      - name: Get Logs from apisix-ingress-controller
         shell: bash
         run: |
           export KUBECONFIG=/tmp/apisix-ingress-cluster.kubeconfig
           kubectl logs -n apisix-conformance-test -l app=apisix-ingress-controller
 
       - name: Upload Gateway API Conformance Report
-        if: ${{ github.event_name == 'push' }}
         uses: actions/upload-artifact@v4
         with:
           name: apisix-ingress-controller-conformance-report-${{ matrix.provider_type }}.yaml
-          path: apisix-ingress-controller-conformance-report.yaml
+          path: ${{ env.CONFORMANCE_TEST_REPORT_OUTPUT }}
 
       - name: Format Conformance Test Report
         if: ${{ github.event_name == 'pull_request' }}
         run: |
           echo '# conformance test report - ${{ matrix.provider_type }} mode' > report.md
           echo '```yaml' >> report.md
-          cat apisix-ingress-controller-conformance-report.yaml >> report.md
+          cat ${CONFORMANCE_TEST_REPORT_OUTPUT} >> report.md
           echo '```' >> report.md
 
       - name: Report Conformance Test Result to PR Comment

.github/workflows/conformance-test.yml

@@ -31,6 +31,8 @@ concurrency:
 
 jobs:
   conformance-test:
+    env:
+      CONFORMANCE_TEST_REPORT_OUTPUT: /tmp/api7-ingress-controller-conformance-report.yaml
     timeout-minutes: 60
     runs-on: ubuntu-latest
     steps:
@@ -105,8 +107,13 @@ jobs:
           API7_EE_LICENSE: ${{ secrets.API7_EE_LICENSE }}
         continue-on-error: true
         run: |
-          make conformance-test
+          make conformance-test-api7ee
       
+      - name: Show Conformance Report
+        shell: bash
+        run: |
+          cat ${CONFORMANCE_TEST_REPORT_OUTPUT}
+
       - name: Get Logs from api7-ingress-controller
         shell: bash
         run: |
@@ -117,20 +124,20 @@ jobs:
         if: ${{ github.event_name == 'push' }}
         uses: actions/upload-artifact@v4
         with:
-          name: apisix-ingress-controller-conformance-report.yaml
-          path: apisix-ingress-controller-conformance-report.yaml
+          name: api7-ingress-controller-conformance-report.yaml
+          path: ${{ env.CONFORMANCE_TEST_REPORT_OUTPUT }}
 
       - name: Format Conformance Test Report
         if: ${{ github.event_name == 'pull_request' }}
         run: |
           echo '# conformance test report' > report.md
           echo '```yaml' >> report.md
-          cat apisix-ingress-controller-conformance-report.yaml >> report.md
+          cat ${CONFORMANCE_TEST_REPORT_OUTPUT} >> report.md
           echo '```' >> report.md
 
       - name: Report Conformance Test Result to PR Comment
         if: ${{ github.event_name == 'pull_request' }}
-        uses: mshick/add-pr-comment@v2
+        uses: ./.github/actions/add-pr-comment
         with:
           message-id: 'conformance-test-report'
           message-path: |

Makefile

@@ -29,10 +29,11 @@ ENVTEST_K8S_VERSION = 1.30.0
 KIND_NAME ?= apisix-ingress-cluster
 KIND_NODE_IMAGE ?= kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e
 
-GATEAY_API_VERSION ?= v1.3.0
 DASHBOARD_VERSION ?= dev
 ADC_VERSION ?= 0.21.0
 
+DIR := $(shell pwd)
+
 GINKGO_VERSION ?= 2.20.0
 TEST_TIMEOUT ?= 80m
 TEST_DIR ?= ./test/e2e/
@@ -55,6 +56,14 @@ GITSHASYM="github.com/apache/apisix-ingress-controller/internal/version._buildGi
 BUILDOSSYM="github.com/apache/apisix-ingress-controller/internal/version._buildOS"
 GO_LDFLAGS ?= "-X=$(VERSYM)=$(VERSION) -X=$(GITSHASYM)=$(GITSHA) -X=$(BUILDOSSYM)=$(OSNAME)/$(OSARCH)"
 
+# gateway-api
+GATEAY_API_VERSION ?= v1.3.0
+## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/pkg/features/httproute.go
+SUPPORTED_EXTENDED_FEATURES = "HTTPRouteDestinationPortMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteRequestMirror,HTTPRouteSchemeRedirect,GatewayAddressEmpty,HTTPRouteResponseHeaderModification,GatewayPort8080"
+CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-report.yaml
+## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go
+CONFORMANCE_PROFILES ?= GATEWAY-HTTP
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -154,12 +163,18 @@ install-ginkgo:
 
 .PHONY: conformance-test
 conformance-test:
-	DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance -tags=conformance -timeout 60m
-
-.PHONY: conformance-test-standalone
-conformance-test-standalone:
-	@kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG
-	go test -v ./test/conformance/apisix -tags=conformance -timeout 60m
+	go test -v ./test/conformance -tags conformance,experimental -timeout 60m \
+		--supported-features=$(SUPPORTED_EXTENDED_FEATURES) \
+		--conformance-profiles=$(CONFORMANCE_PROFILES) \
+		--report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT)
+
+
+.PHONY: conformance-test-api7ee
+conformance-test-api7ee:
+	DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance/api7ee -tags conformance,experimental -timeout 60m \
+		--supported-features=$(SUPPORTED_EXTENDED_FEATURES) \
+		--conformance-profiles=$(CONFORMANCE_PROFILES) \
+		--report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT)
 
 .PHONY: lint
 lint: sort-import golangci-lint ## Run golangci-lint linter

test/conformance/apisix/conformance_test.go renamed to test/conformance/api7ee/conformance_test.go

@@ -15,13 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build conformance
-// +build conformance
-
-package conformance
+package api7ee
 
 import (
-	"flag"
 	"os"
 	"testing"
 
@@ -30,8 +26,8 @@ import (
 	"sigs.k8s.io/gateway-api/conformance"
 	conformancev1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/tests"
+	"sigs.k8s.io/gateway-api/conformance/utils/flags"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
-	"sigs.k8s.io/gateway-api/pkg/features"
 	"sigs.k8s.io/yaml"
 )
 
@@ -43,24 +39,11 @@ var skippedTestsForSSL = []string{
 
 // TODO: HTTPRoute hostname intersection and listener hostname matching
 
-var gatewaySupportedFeatures = []features.FeatureName{
-	features.SupportGateway,
-	features.SupportHTTPRoute,
-	// features.SupportHTTPRouteMethodMatching,
-	// features.SupportHTTPRouteResponseHeaderModification,
-	// features.SupportHTTPRouteRequestMirror,
-	// features.SupportHTTPRouteBackendRequestHeaderModification,
-	// features.SupportHTTPRouteHostRewrite,
-}
-
 func TestGatewayAPIConformance(t *testing.T) {
-	flag.Parse()
-
 	opts := conformance.DefaultOptions(t)
 	opts.Debug = true
 	opts.CleanupBaseResources = true
 	opts.GatewayClassName = gatewayClassName
-	opts.SupportedFeatures = sets.New(gatewaySupportedFeatures...)
 	opts.SkipTests = skippedTestsForSSL
 	opts.Implementation = conformancev1.Implementation{
 		Organization: "APISIX",
@@ -80,7 +63,6 @@ func TestGatewayAPIConformance(t *testing.T) {
 		t.Fatalf("failed to run the gateway conformance test suite: %v", err)
 	}
 
-	const reportFileName = "apisix-ingress-controller-conformance-report.yaml"
 	report, err := cSuite.Report()
 	if err != nil {
 		t.Fatalf("failed to get the gateway conformance test report: %v", err)
@@ -90,6 +72,10 @@ func TestGatewayAPIConformance(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to marshal the gateway conformance test report: %v", err)
 	}
-	// Save report in the root of the repository, file name is in .gitignore.
-	require.NoError(t, os.WriteFile("../../../"+reportFileName, rawReport, 0o600))
+	f, err := os.Create(*flags.ReportOutput)
+	require.NoError(t, err)
+	defer func() { _ = f.Close() }()
+
+	_, err = f.Write(rawReport)
+	require.NoError(t, err)
 }

test/conformance/apisix/suite_test.go renamed to test/conformance/api7ee/suite_test.go

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package conformance
+package api7ee
 
 import (
 	"context"
@@ -32,7 +32,6 @@ import (
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	"github.com/apache/apisix-ingress-controller/test/e2e/framework"
-	"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
 )
 
 var gatewayClassName = "apisix"
@@ -124,8 +123,7 @@ func TestMain(m *testing.M) {
 	RegisterFailHandler(Fail)
 	f := framework.NewFramework()
 
-	// init newDeployer function
-	scaffold.NewDeployer = scaffold.NewAPISIXDeployer
+	f.BeforeSuite()
 
 	// Check and delete specific namespaces if they exist
 	kubectl := k8s.NewKubectlOptions("", "", "default")
@@ -138,51 +136,49 @@ func TestMain(m *testing.M) {
 	k8s.CreateNamespace(GinkgoT(), kubectl, namespace)
 	defer k8s.DeleteNamespace(GinkgoT(), kubectl, namespace)
 
-	adminkey := getEnvOrDefault("APISIX_ADMIN_KEY", "edd1c9f034335f136f87ad84b625c8f1")
-	controllerName := "apisix.apache.org/apisix-ingress-controller"
-	s := scaffold.NewScaffold(scaffold.Options{
-		ControllerName:    controllerName,
-		SkipHooks:         true,
-		APISIXAdminAPIKey: adminkey,
+	gatewayGroupId := f.CreateNewGatewayGroupWithIngress()
+	adminKey := f.GetAdminKey(gatewayGroupId)
+
+	svc := f.DeployGateway(&framework.API7DeployOptions{
+		Namespace:              namespace,
+		GatewayGroupID:         gatewayGroupId,
+		DPManagerEndpoint:      framework.DPManagerTLSEndpoint,
+		SetEnv:                 true,
+		SSLKey:                 framework.TestKey,
+		SSLCert:                framework.TestCert,
+		TLSEnabled:             true,
+		ForIngressGatewayGroup: true,
+		ServiceType:            "LoadBalancer",
+		ServiceHTTPPort:        80,
+		ServiceHTTPSPort:       443,
 	})
 
-	s.Deployer.DeployDataplane(scaffold.DeployDataplaneOptions{
-		AdminKey:          adminkey,
-		Namespace:         namespace,
-		SkipCreateTunnels: true,
-		ServiceType:       "LoadBalancer",
-		ServiceHTTPPort:   80,
-		ServiceHTTPSPort:  443,
-	})
-	svc := s.GetDataplaneService()
-
 	if len(svc.Status.LoadBalancer.Ingress) == 0 {
 		Fail("No LoadBalancer found for the service")
 	}
 
 	address := svc.Status.LoadBalancer.Ingress[0].IP
 
 	f.DeployIngress(framework.IngressDeployOpts{
-		ControllerName:     controllerName,
-		Namespace:          namespace,
-		StatusAddress:      address,
-		InitSyncDelay:      20 * time.Minute,
-		ProviderType:       framework.ProviderType,
-		ProviderSyncPeriod: 1 * time.Hour,
+		ControllerName: "apisix.apache.org/apisix-ingress-controller",
+		Namespace:      namespace,
+		StatusAddress:  address,
+		InitSyncDelay:  1 * time.Minute,
+		ProviderType:   "api7ee",
 	})
 
-	adminEndpoint := fmt.Sprintf("http://%s.%s:9180", svc.Name, namespace)
-
 	defaultGatewayProxyOpts = GatewayProxyOpts{
 		StatusAddress: address,
-		AdminKey:      adminkey,
-		AdminEndpoint: adminEndpoint,
+		AdminKey:      adminKey,
+		AdminEndpoint: framework.DashboardTLSEndpoint,
 	}
 
 	patchGatewaysForConformanceTest(context.Background(), f.K8sClient)
 
 	code := m.Run()
 
+	f.AfterSuite()
+
 	os.Exit(code)
 }
 
@@ -262,11 +258,3 @@ func patchGatewaysForConformanceTest(ctx context.Context, k8sClient client.Clien
 		}
 	}()
 }
-
-// getEnvOrDefault returns environment variable value or default
-func getEnvOrDefault(key, defaultValue string) string {
-	if value := os.Getenv(key); value != "" {
-		return value
-	}
-	return defaultValue
-}