chore: add more conformance-test report for gateway-api (#2557)

Author: AlinsRan

.github/workflows/apisix-conformance-test.yml

@@ -35,27 +35,10 @@ permissions:
   pull-requests: write
 
 jobs:
-  prepare:
-    name: Prepare
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Go Env
-        id: go
-        uses: actions/setup-go@v4
-        with:
-          go-version: "1.24"
-
-      - name: Install kind
-        run: |
-          go install sigs.k8s.io/[email protected]
-
   conformance-test:
+    env:
+      CONFORMANCE_TEST_REPORT_OUTPUT: /tmp/apisix-ingress-controller-conformance-report.yaml
     timeout-minutes: 60
-    needs: 
-      - prepare
     strategy:
       matrix:
         provider_type:
@@ -73,6 +56,10 @@ jobs:
         with:
           go-version: "1.24"
 
+      - name: Install kind
+        run: |
+          go install sigs.k8s.io/[email protected]
+
       - name: Build images
         env:
           TAG: dev
@@ -107,24 +94,20 @@ jobs:
         env:
           PROVIDER_TYPE: ${{ matrix.provider_type }}
         run: |
-          make conformance-test-standalone
+          make conformance-test
       
+      - name: Show Conformance Report
+        shell: bash
+        run: |
+          cat ${CONFORMANCE_TEST_REPORT_OUTPUT}
+
       - name: Get Logs from apisix-ingress-controller
         shell: bash
         run: |
           kubectl logs -n apisix-conformance-test -l app=apisix-ingress-controller
 
       - name: Upload Gateway API Conformance Report
-        if: ${{ github.event_name == 'push' }}
         uses: actions/upload-artifact@v4
         with:
           name: apisix-ingress-controller-conformance-report-${{ matrix.provider_type }}.yaml
-          path: apisix-ingress-controller-conformance-report.yaml
-
-      - name: Format Conformance Test Report
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo '# conformance test report - ${{ matrix.provider_type }} mode' > report.md
-          echo '```yaml' >> report.md
-          cat apisix-ingress-controller-conformance-report.yaml >> report.md
-          echo '```' >> report.md
+          path: ${{ env.CONFORMANCE_TEST_REPORT_OUTPUT }}

Makefile

@@ -27,9 +27,10 @@ IMG ?= apache/apisix-ingress-controller:$(IMAGE_TAG)
 ENVTEST_K8S_VERSION = 1.30.0
 KIND_NAME ?= apisix-ingress-cluster
 
-GATEAY_API_VERSION ?= v1.3.0
 ADC_VERSION ?= 0.21.0
 
+DIR := $(shell pwd)
+
 GINKGO_VERSION ?= 2.20.0
 TEST_TIMEOUT ?= 80m
 TEST_DIR ?= ./test/e2e/apisix/
@@ -48,6 +49,14 @@ GITSHASYM="github.com/apache/apisix-ingress-controller/internal/version._buildGi
 BUILDOSSYM="github.com/apache/apisix-ingress-controller/internal/version._buildOS"
 GO_LDFLAGS ?= "-X=$(VERSYM)=$(VERSION) -X=$(GITSHASYM)=$(GITSHA) -X=$(BUILDOSSYM)=$(OSNAME)/$(OSARCH)"
 
+# gateway-api
+GATEAY_API_VERSION ?= v1.3.0
+## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/pkg/features/httproute.go
+SUPPORTED_EXTENDED_FEATURES = "HTTPRouteDestinationPortMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteRequestMirror,HTTPRouteSchemeRedirect,GatewayAddressEmpty,HTTPRouteResponseHeaderModification,GatewayPort8080"
+CONFORMANCE_TEST_REPORT_OUTPUT ?= $(DIR)/apisix-ingress-controller-conformance-report.yaml
+## https://github.com/kubernetes-sigs/gateway-api/blob/v1.3.0/conformance/utils/suite/profiles.go
+CONFORMANCE_PROFILES ?= GATEWAY-HTTP
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -139,11 +148,10 @@ install-ginkgo:
 
 .PHONY: conformance-test
 conformance-test:
-	go test -v ./test/conformance -tags=conformance -timeout 60m
-
-.PHONY: conformance-test-standalone
-conformance-test-standalone:
-	go test -v ./test/conformance/apisix -tags=conformance -timeout 60m
+	go test -v ./test/conformance -tags conformance,experimental -timeout 60m \
+		--supported-features=$(SUPPORTED_EXTENDED_FEATURES) \
+		--conformance-profiles=$(CONFORMANCE_PROFILES) \
+		--report-output=$(CONFORMANCE_TEST_REPORT_OUTPUT)
 
 .PHONY: lint
 lint: sort-import golangci-lint ## Run golangci-lint linter

…t/conformance/apisix/conformance_test.go test/conformance/conformance_test.gotest/conformance/apisix/conformance_test.go renamed to test/conformance/conformance_test.go test/conformance/apisix/conformance_test.go renamed to test/conformance/conformance_test.go

@@ -15,60 +15,41 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//go:build conformance
-// +build conformance
-
 package conformance
 
 import (
-	"flag"
 	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
-	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/gateway-api/conformance"
 	conformancev1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/tests"
+	"sigs.k8s.io/gateway-api/conformance/utils/flags"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
-	"sigs.k8s.io/gateway-api/pkg/features"
 	"sigs.k8s.io/yaml"
 )
 
+// https://github.com/kubernetes-sigs/gateway-api/blob/5c5fc388829d24e8071071b01e8313ada8f15d9f/conformance/utils/suite/suite.go#L358.  SAN includes '*'
 var skippedTestsForSSL = []string{
-	// Reason: https://github.com/kubernetes-sigs/gateway-api/blob/5c5fc388829d24e8071071b01e8313ada8f15d9f/conformance/utils/suite/suite.go#L358.  SAN includes '*'
 	tests.HTTPRouteHTTPSListener.ShortName,
 	tests.HTTPRouteRedirectPortAndScheme.ShortName,
 }
 
 // TODO: HTTPRoute hostname intersection and listener hostname matching
 
-var gatewaySupportedFeatures = []features.FeatureName{
-	features.SupportGateway,
-	features.SupportHTTPRoute,
-	// features.SupportHTTPRouteMethodMatching,
-	// features.SupportHTTPRouteResponseHeaderModification,
-	// features.SupportHTTPRouteRequestMirror,
-	// features.SupportHTTPRouteBackendRequestHeaderModification,
-	// features.SupportHTTPRouteHostRewrite,
-}
-
 func TestGatewayAPIConformance(t *testing.T) {
-	flag.Parse()
-
 	opts := conformance.DefaultOptions(t)
 	opts.Debug = true
 	opts.CleanupBaseResources = true
 	opts.GatewayClassName = gatewayClassName
-	opts.SupportedFeatures = sets.New(gatewaySupportedFeatures...)
 	opts.SkipTests = skippedTestsForSSL
 	opts.Implementation = conformancev1.Implementation{
 		Organization: "APISIX",
 		Project:      "apisix-ingress-controller",
 		URL:          "https://github.com/apache/apisix-ingress-controller.git",
 		Version:      "v2.0.0",
 	}
-	opts.ConformanceProfiles = sets.New(suite.GatewayHTTPConformanceProfileName)
 
 	cSuite, err := suite.NewConformanceTestSuite(opts)
 	require.NoError(t, err)
@@ -80,7 +61,6 @@ func TestGatewayAPIConformance(t *testing.T) {
 		t.Fatalf("failed to run the gateway conformance test suite: %v", err)
 	}
 
-	const reportFileName = "apisix-ingress-controller-conformance-report.yaml"
 	report, err := cSuite.Report()
 	if err != nil {
 		t.Fatalf("failed to get the gateway conformance test report: %v", err)
@@ -90,6 +70,10 @@ func TestGatewayAPIConformance(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to marshal the gateway conformance test report: %v", err)
 	}
-	// Save report in the root of the repository, file name is in .gitignore.
-	require.NoError(t, os.WriteFile("../../../"+reportFileName, rawReport, 0o600))
+	f, err := os.Create(*flags.ReportOutput)
+	require.NoError(t, err)
+	defer func() { _ = f.Close() }()
+
+	_, err = f.Write(rawReport)
+	require.NoError(t, err)
 }