Merge pull request #2633 from programmer04/sort-report

Sort items in conformance reports to maintain stability

Author: k8s-ci-robot

conformance/utils/suite/experimental_reports.go

@@ -17,6 +17,8 @@ limitations under the License.
 package suite
 
 import (
+	"sort"
+
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
@@ -72,9 +74,6 @@ func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile,
 			if report.Extended == nil {
 				report.Extended = &confv1a1.ExtendedStatus{}
 			}
-			if report.Extended.FailedTests == nil {
-				report.Extended.FailedTests = []string{}
-			}
 			report.Extended.FailedTests = append(report.Extended.FailedTests, result.test.ShortName)
 			report.Extended.Statistics.Failed++
 		} else {
@@ -90,15 +89,9 @@ func (p profileReportsMap) addTestResults(conformanceProfile ConformanceProfile,
 				report.Extended = &confv1a1.ExtendedStatus{}
 			}
 			report.Extended.Statistics.Skipped++
-			if report.Extended.SkippedTests == nil {
-				report.Extended.SkippedTests = []string{}
-			}
 			report.Extended.SkippedTests = append(report.Extended.SkippedTests, result.test.ShortName)
 		} else {
 			report.Core.Statistics.Skipped++
-			if report.Core.SkippedTests == nil {
-				report.Core.SkippedTests = []string{}
-			}
 			report.Core.SkippedTests = append(report.Core.SkippedTests, result.test.ShortName)
 		}
 	}
@@ -140,10 +133,11 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
 		supportedFeatures := supportedFeaturesMap[ConformanceProfileName(report.Name)]
 		if report.Extended != nil {
 			if supportedFeatures != nil {
-				if report.Extended.SupportedFeatures == nil {
-					report.Extended.SupportedFeatures = make([]string, 0)
-				}
-				for _, f := range supportedFeatures.UnsortedList() {
+				supportedFeatures := supportedFeatures.UnsortedList()
+				sort.Slice(supportedFeatures, func(i, j int) bool {
+					return supportedFeatures[i] < supportedFeatures[j]
+				})
+				for _, f := range supportedFeatures {
 					report.Extended.SupportedFeatures = append(report.Extended.SupportedFeatures, string(f))
 				}
 			}
@@ -152,10 +146,11 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
 		unsupportedFeatures := unsupportedFeaturesMap[ConformanceProfileName(report.Name)]
 		if report.Extended != nil {
 			if unsupportedFeatures != nil {
-				if report.Extended.UnsupportedFeatures == nil {
-					report.Extended.UnsupportedFeatures = make([]string, 0)
-				}
-				for _, f := range unsupportedFeatures.UnsortedList() {
+				unsupportedFeatures := unsupportedFeatures.UnsortedList()
+				sort.Slice(unsupportedFeatures, func(i, j int) bool {
+					return unsupportedFeatures[i] < unsupportedFeatures[j]
+				})
+				for _, f := range unsupportedFeatures {
 					report.Extended.UnsupportedFeatures = append(report.Extended.UnsupportedFeatures, string(f))
 				}
 			}

conformance/utils/suite/experimental_suite.go

@@ -19,6 +19,7 @@ package suite
 import (
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"sync"
 	"testing"
@@ -255,10 +256,19 @@ func (suite *ExperimentalConformanceTestSuite) Report() (*confv1a1.ConformanceRe
 	}
 	defer suite.lock.RUnlock()
 
+	testNames := make([]string, 0, len(suite.results))
+	for tN := range suite.results {
+		testNames = append(testNames, tN)
+	}
+	sort.Strings(testNames)
 	profileReports := newReports()
-	for _, testResult := range suite.results {
-		conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles)
-		for _, profile := range conformanceProfiles.UnsortedList() {
+	for _, tN := range testNames {
+		testResult := suite.results[tN]
+		conformanceProfiles := getConformanceProfilesForTest(testResult.test, suite.conformanceProfiles).UnsortedList()
+		sort.Slice(conformanceProfiles, func(i, j int) bool {
+			return conformanceProfiles[i].Name < conformanceProfiles[j].Name
+		})
+		for _, profile := range conformanceProfiles {
 			profileReports.addTestResults(*profile, testResult)
 		}
 	}