Infer SupportedFeatures in Conformance Tests (GEP-2162) [#3759] (#3848)

* SupportedFeatures

* Added inferredSupportedFeatures to determine and report if feature for tests were manually supplied or inferred from GatewayClass

* Corrected suite unit tests after addition of a new field.

* Added logic of determining if supportedFeatures are inferred or supplied and refactored some code around it.

* moved SupportedFeatures init after scheme addition

* Cleaned up and organized code to properly determine features we are testing and if they were inferred or not.

* Cleanup v2

* Make inferring supportedFeatures to take precedence over all other feature flags.

* Fix when no profile was setting inferred to false.

* remove debug log

* Switched args arrangement so ctx is the first argument passed to fetchSupportedFeatures function.

* Updated context as it's unclear where it should be supplied from

* removed SupportedFeatures struct and method for determining supported features.

* Refactored logic of determining supported features and add flag for the report in cSuite.

* Reversed flag parsing for some values.

* Tweaked logging options.

* Cleaned up last places for SupportedFeatures struct.

* Wrote unit tests for supported features determination logic in conformance suite.

* Formatting fixes.

* Resolved comments.

* Fixed formatting.

* Updated conformance docs.

* Removed doc update so it can be submitted as a seperate PR.

Author: bexxmodd

conformance/apis/v1/conformancereport.go

@@ -50,6 +50,10 @@ type ConformanceReport struct {
 	// SucceededProvisionalTests is a list of the names of the provisional tests that
 	// have been successfully run.
 	SucceededProvisionalTests []string `json:"succeededProvisionalTests,omitempty"`
+
+	// InferredSupportedFeatures indicates whether the supported features were
+	// automatically detected by the conformance suite.
+	InferredSupportedFeatures bool `json:"inferredSupportedFeatures"`
 }
 
 // Implementation provides metadata information on the downstream

conformance/conformance.go

@@ -29,11 +29,14 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/tests"
 	conformanceconfig "sigs.k8s.io/gateway-api/conformance/utils/config"
 	"sigs.k8s.io/gateway-api/conformance/utils/flags"
+
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 
 	"github.com/stretchr/testify/require"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+
 	clientset "k8s.io/client-go/kubernetes"
+
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/yaml"
@@ -63,6 +66,7 @@ func DefaultOptions(t *testing.T) suite.ConformanceOptions {
 
 	supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures)
 	exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures)
+
 	skipTests := suite.ParseSkipTests(*flags.SkipTests)
 	namespaceLabels := suite.ParseKeyValuePairs(*flags.NamespaceLabels)
 	namespaceAnnotations := suite.ParseKeyValuePairs(*flags.NamespaceAnnotations)
@@ -144,6 +148,7 @@ func logOptions(t *testing.T, opts suite.ConformanceOptions) {
 	t.Logf("  Enable All Features: %t", opts.EnableAllSupportedFeatures)
 	t.Logf("  Supported Features: %v", opts.SupportedFeatures.UnsortedList())
 	t.Logf("  ExemptFeatures: %v", opts.ExemptFeatures.UnsortedList())
+	t.Logf("  ConformanceProfiles: %v", opts.ConformanceProfiles.UnsortedList())
 }
 
 func writeReport(logf func(string, ...any), report confv1.ConformanceReport, output string) error {

conformance/utils/suite/conformance.go

@@ -21,8 +21,6 @@ import (
 	"strings"
 	"testing"
 
-	"k8s.io/apimachinery/pkg/util/sets"
-
 	"sigs.k8s.io/gateway-api/conformance/utils/tlog"
 	"sigs.k8s.io/gateway-api/pkg/features"
 )
@@ -81,11 +79,11 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
 
 // ParseSupportedFeatures parses flag arguments and converts the string to
 // sets.Set[features.FeatureName]
-func ParseSupportedFeatures(f string) sets.Set[features.FeatureName] {
+func ParseSupportedFeatures(f string) FeaturesSet {
 	if f == "" {
 		return nil
 	}
-	res := sets.Set[features.FeatureName]{}
+	res := FeaturesSet{}
 	for _, value := range strings.Split(f, ",") {
 		res.Insert(features.FeatureName(value))
 	}

conformance/utils/suite/reports.go

@@ -22,7 +22,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
-	"sigs.k8s.io/gateway-api/pkg/features"
 )
 
 // -----------------------------------------------------------------------------
@@ -107,7 +106,7 @@ func (p profileReportsMap) list() (profileReports []confv1.ProfileReport) {
 	return
 }
 
-func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformanceProfileName]sets.Set[features.FeatureName], unsupportedFeaturesMap map[ConformanceProfileName]sets.Set[features.FeatureName]) {
+func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformanceProfileName]FeaturesSet, unsupportedFeaturesMap map[ConformanceProfileName]FeaturesSet) {
 	for key, report := range p {
 		// report the overall result for core features
 		switch {
@@ -162,7 +161,8 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
 // isTestExtended determines if a provided test is considered to be supported
 // at an extended level of support given the provided conformance profile.
 //
-// TODO: right now the tests themselves don't indicate the conformance
+// TODO(#3759) Update this method to be based on Features inferred.
+// Right now the tests themselves don't indicate the conformance
 // support level associated with them. The only way we have right now
 // in this prototype to know whether a test belongs to any particular
 // conformance level is to compare the features needed for the test to

conformance/utils/suite/suite.go

@@ -31,11 +31,13 @@ import (
 	"github.com/stretchr/testify/require"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
 	confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/utils/config"
@@ -69,7 +71,7 @@ type ConformanceTestSuite struct {
 	BaseManifests            string
 	MeshManifests            string
 	Applier                  kubernetes.Applier
-	SupportedFeatures        sets.Set[features.FeatureName]
+	SupportedFeatures        FeaturesSet
 	TimeoutConfig            config.TimeoutConfig
 	SkipTests                sets.Set[string]
 	SkipProvisionalTests     bool
@@ -79,6 +81,11 @@ type ConformanceTestSuite struct {
 	UsableNetworkAddresses   []v1beta1.GatewaySpecAddress
 	UnusableNetworkAddresses []v1beta1.GatewaySpecAddress
 
+	// If SupportedFeatures are automatically determined from GWC Status.
+	// This will be required to report in future iterations as the passing
+	// will be determined based on this.
+	isInferredSupportedFeatures bool
+
 	// mode is the operating mode of the implementation.
 	// The default value for it is "default".
 	mode string
@@ -142,8 +149,8 @@ type ConformanceOptions struct {
 	// CleanupBaseResources indicates whether or not the base test
 	// resources such as Gateways should be cleaned up after the run.
 	CleanupBaseResources       bool
-	SupportedFeatures          sets.Set[features.FeatureName]
-	ExemptFeatures             sets.Set[features.FeatureName]
+	SupportedFeatures          FeaturesSet
+	ExemptFeatures             FeaturesSet
 	EnableAllSupportedFeatures bool
 	TimeoutConfig              config.TimeoutConfig
 	// SkipTests contains all the tests not to be run and can be used to opt out
@@ -172,6 +179,8 @@ type ConformanceOptions struct {
 	ConformanceProfiles sets.Set[ConformanceProfileName]
 }
 
+type FeaturesSet = sets.Set[features.FeatureName]
+
 const (
 	// undefinedKeyword is set in the ConformanceReport "GatewayAPIVersion" and
 	// "GatewayAPIChannel" fields in case it's not possible to figure out the actual
@@ -181,16 +190,23 @@ const (
 
 // NewConformanceTestSuite is a helper to use for creating a new ConformanceTestSuite.
 func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite, error) {
-	// test suite callers are required to provide either:
-	// - one conformance profile via the flag '-conformance-profiles'
-	// - a list of supported features via the flag '-supported-features'
-	// - an explicit test to run via the flag '-run-test'
-	// - all features are being tested via the flag '-all-features'
-	if options.SupportedFeatures.Len() == 0 &&
-		options.ConformanceProfiles.Len() == 0 &&
-		!options.EnableAllSupportedFeatures &&
-		options.RunTest == "" {
-		return nil, fmt.Errorf("no conformance profile, supported features, explicit tests were provided so no tests could be selected")
+	supportedFeatures := options.SupportedFeatures.Difference(options.ExemptFeatures)
+	isInferred := false
+	switch {
+	case options.EnableAllSupportedFeatures:
+		supportedFeatures = features.SetsToNamesSet(features.AllFeatures)
+	case shouldInferSupportedFeatures(&options):
+		var err error
+		supportedFeatures, err = fetchSupportedFeatures(options.Client, options.GatewayClassName)
+		if err != nil {
+			return nil, fmt.Errorf("Cannot infer supported features: %w", err)
+		}
+		isInferred = true
+	}
+
+	// If features were not inferred from Status, it's a GWC issue.
+	if isInferred && supportedFeatures.Len() == 0 {
+		return nil, fmt.Errorf("no supported features were determined for test suite")
 	}
 
 	config.SetupTimeoutConfig(&options.TimeoutConfig)
@@ -224,19 +240,6 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
 		mode = options.Mode
 	}
 
-	// test suite callers can potentially just run all tests by saying they
-	// cover all features, if they don't they'll need to have provided a
-	// conformance profile or at least some specific features they support.
-	if options.EnableAllSupportedFeatures {
-		options.SupportedFeatures = features.SetsToNamesSet(features.AllFeatures)
-	} else if options.SupportedFeatures == nil {
-		options.SupportedFeatures = sets.New[features.FeatureName]()
-	}
-
-	for feature := range options.ExemptFeatures {
-		options.SupportedFeatures.Delete(feature)
-	}
-
 	suite := &ConformanceTestSuite{
 		Client:           options.Client,
 		ClientOptions:    options.ClientOptions,
@@ -254,7 +257,7 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
 			NamespaceAnnotations: options.NamespaceAnnotations,
 			AddressType:          options.AddressType,
 		},
-		SupportedFeatures:           options.SupportedFeatures,
+		SupportedFeatures:           supportedFeatures,
 		TimeoutConfig:               options.TimeoutConfig,
 		SkipTests:                   sets.New(options.SkipTests...),
 		RunTest:                     options.RunTest,
@@ -270,6 +273,7 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
 		mode:                        mode,
 		apiVersion:                  apiVersion,
 		apiChannel:                  apiChannel,
+		isInferredSupportedFeatures: isInferred,
 		Hook:                        options.Hook,
 	}
 
@@ -288,12 +292,12 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
 		for _, f := range conformanceProfile.ExtendedFeatures.UnsortedList() {
 			if options.SupportedFeatures.Has(f) {
 				if suite.extendedSupportedFeatures[conformanceProfileName] == nil {
-					suite.extendedSupportedFeatures[conformanceProfileName] = sets.New[features.FeatureName]()
+					suite.extendedSupportedFeatures[conformanceProfileName] = FeaturesSet{}
 				}
 				suite.extendedSupportedFeatures[conformanceProfileName].Insert(f)
 			} else {
 				if suite.extendedUnsupportedFeatures[conformanceProfileName] == nil {
-					suite.extendedUnsupportedFeatures[conformanceProfileName] = sets.New[features.FeatureName]()
+					suite.extendedUnsupportedFeatures[conformanceProfileName] = FeaturesSet{}
 				}
 				suite.extendedUnsupportedFeatures[conformanceProfileName].Insert(f)
 			}
@@ -390,6 +394,10 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T, tests []ConformanceTest)
 	}
 }
 
+func (suite *ConformanceTestSuite) IsInferredSupportedFeatures() bool {
+	return suite.isInferredSupportedFeatures
+}
+
 func (suite *ConformanceTestSuite) setClientsetForTest(test ConformanceTest) error {
 	featureNames := []string{}
 	for _, v := range test.Features {
@@ -544,6 +552,7 @@ func (suite *ConformanceTestSuite) Report() (*confv1.ConformanceReport, error) {
 		GatewayAPIChannel:         suite.apiChannel,
 		ProfileReports:            profileReports.list(),
 		SucceededProvisionalTests: succeededProvisionalTests,
+		InferredSupportedFeatures: suite.IsInferredSupportedFeatures(),
 	}, nil
 }
 
@@ -573,6 +582,39 @@ func ParseConformanceProfiles(p string) sets.Set[ConformanceProfileName] {
 	return res
 }
 
+func fetchSupportedFeatures(client client.Client, gatewayClassName string) (FeaturesSet, error) {
+	if gatewayClassName == "" {
+		return nil, fmt.Errorf("GatewayClass name must be provided to fetch supported features")
+	}
+	gwc := &gatewayv1.GatewayClass{}
+	err := client.Get(context.TODO(), types.NamespacedName{Name: gatewayClassName}, gwc)
+	if err != nil {
+		return nil, fmt.Errorf("fetchSupportedFeatures(): %w", err)
+	}
+
+	fs := FeaturesSet{}
+	for _, feature := range gwc.Status.SupportedFeatures {
+		fs.Insert(features.FeatureName(feature.Name))
+	}
+	fmt.Printf("Supported features for GatewayClass %s: %v\n", gatewayClassName, fs.UnsortedList())
+	return fs, nil
+}
+
+// shouldInferSupportedFeatures checks if any flags were supplied for manually
+// picking what to test. Inferred supported features are only used when no flags
+// are set.
+func shouldInferSupportedFeatures(opts *ConformanceOptions) bool {
+	if opts == nil {
+		return false
+	}
+	return !opts.EnableAllSupportedFeatures &&
+		opts.SupportedFeatures.Len() == 0 &&
+		opts.ExemptFeatures.Len() == 0 &&
+		opts.ConformanceProfiles.Len() == 0 &&
+		len(opts.SkipTests) == 0 &&
+		opts.RunTest == ""
+}
+
 // getAPIVersionAndChannel iterates over all the crds installed in the cluster and check the version and channel annotations.
 // In case the annotations are not found or there are crds with different versions or channels, an error is returned.
 func getAPIVersionAndChannel(crds []apiextensionsv1.CustomResourceDefinition) (version string, channel string, err error) {