Merge pull request #2799 from mlavacca/report-summary

feat: conformance report summary field

Author: k8s-ci-robot

conformance/apis/v1alpha1/profilereport.go

@@ -23,6 +23,10 @@ type ProfileReport struct {
 	// "TLS", "Mesh", e.t.c.).
 	Name string `json:"name"`
 
+	// Summary is a human-readable message intended for end-users to understand
+	// the overall status at a glance.
+	Summary string `json:"summary"`
+
 	// Core indicates the core support level which includes the set of tests
 	// which are the minimum the implementation must pass to be considered at
 	// all conformant.
@@ -51,10 +55,6 @@ type ExtendedStatus struct {
 type Status struct {
 	Result `json:"result"`
 
-	// Summary is a human-readable message intended for end-users to understand
-	// the overall status at a glance.
-	Summary string `json:"summary"`
-
 	// Statistics includes numerical statistics on the result of the test run.
 	Statistics `json:"statistics"`
 

conformance/experimental_conformance_test.go

@@ -27,7 +27,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/yaml"
 
-	v1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 	"sigs.k8s.io/gateway-api/apis/v1alpha2"
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
 	confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
@@ -66,7 +66,7 @@ func TestExperimentalConformance(t *testing.T) {
 
 	v1alpha2.AddToScheme(mgrClient.Scheme())
 	v1beta1.AddToScheme(mgrClient.Scheme())
-	v1.AddToScheme(mgrClient.Scheme())
+	gatewayv1.AddToScheme(mgrClient.Scheme())
 
 	// standard conformance flags
 	supportedFeatures = suite.ParseSupportedFeatures(*flags.SupportedFeatures)

conformance/utils/suite/experimental_reports.go

@@ -17,6 +17,7 @@ limitations under the License.
 package suite
 
 import (
+	"fmt"
 	"sort"
 
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -128,6 +129,7 @@ func (p profileReportsMap) compileResults(supportedFeaturesMap map[ConformancePr
 				report.Extended.Result = confv1a1.Success
 			}
 		}
+		report.Summary = buildSummary(report)
 		p[key] = report
 
 		supportedFeatures := supportedFeaturesMap[ConformanceProfileName(report.Name)]
@@ -183,3 +185,25 @@ func isTestExtended(profile ConformanceProfile, test ConformanceTest) bool {
 	}
 	return false
 }
+
+// buildSummary creates a human-readable message about each profile's test outcomes.
+func buildSummary(report confv1a1.ProfileReport) (reportSummary string) {
+	reportSummary = fmt.Sprintf("Core tests %s", buildReportSummary(report.Core))
+	if report.Extended != nil {
+		reportSummary = fmt.Sprintf("%s. Extended tests %s", reportSummary, buildReportSummary(report.Extended.Status))
+	}
+	return fmt.Sprintf("%s.", reportSummary)
+}
+
+func buildReportSummary(status confv1a1.Status) string {
+	var message string
+	switch status.Result {
+	case confv1a1.Success:
+		message = "succeeded"
+	case confv1a1.Partial:
+		message = fmt.Sprintf("partially succeeded with %d test skips", status.Statistics.Skipped)
+	case confv1a1.Failure:
+		message = fmt.Sprintf("failed with %d test failures", status.Statistics.Failed)
+	}
+	return message
+}

conformance/utils/suite/experimental_reports_test.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package suite
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	confv1a1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
+)
+
+func TestBuildSummary(t *testing.T) {
+	testCases := []struct {
+		name            string
+		report          confv1a1.ProfileReport
+		expectedSummary string
+	}{
+		{
+			name: "core tests failed, no extended tests",
+			report: confv1a1.ProfileReport{
+				Name: string(HTTPConformanceProfileName),
+				Core: confv1a1.Status{
+					Result: confv1a1.Failure,
+					Statistics: confv1a1.Statistics{
+						Passed: 5,
+						Failed: 3,
+					},
+				},
+			},
+			expectedSummary: "Core tests failed with 3 test failures.",
+		},
+		{
+			name: "core tests succeeded, extended tests failed",
+			report: confv1a1.ProfileReport{
+				Name: string(HTTPConformanceProfileName),
+				Core: confv1a1.Status{
+					Result: confv1a1.Success,
+					Statistics: confv1a1.Statistics{
+						Passed: 8,
+					},
+				},
+				Extended: &confv1a1.ExtendedStatus{
+					Status: confv1a1.Status{
+						Result: confv1a1.Failure,
+						Statistics: confv1a1.Statistics{
+							Passed: 2,
+							Failed: 1,
+						},
+					},
+				},
+			},
+			expectedSummary: "Core tests succeeded. Extended tests failed with 1 test failures.",
+		},
+		{
+			name: "core tests partially succeeded, extended tests succeeded",
+			report: confv1a1.ProfileReport{
+				Name: string(HTTPConformanceProfileName),
+				Core: confv1a1.Status{
+					Result: confv1a1.Partial,
+					Statistics: confv1a1.Statistics{
+						Passed:  6,
+						Skipped: 2,
+					},
+				},
+				Extended: &confv1a1.ExtendedStatus{
+					Status: confv1a1.Status{
+						Result: confv1a1.Success,
+						Statistics: confv1a1.Statistics{
+							Passed: 2,
+						},
+					},
+				},
+			},
+			expectedSummary: "Core tests partially succeeded with 2 test skips. Extended tests succeeded.",
+		},
+		{
+			name: "core tests succeeded, extended tests partially succeeded",
+			report: confv1a1.ProfileReport{
+				Name: string(HTTPConformanceProfileName),
+				Core: confv1a1.Status{
+					Result: confv1a1.Success,
+					Statistics: confv1a1.Statistics{
+						Passed: 8,
+					},
+				},
+				Extended: &confv1a1.ExtendedStatus{
+					Status: confv1a1.Status{
+						Result: confv1a1.Partial,
+						Statistics: confv1a1.Statistics{
+							Passed:  2,
+							Skipped: 1,
+						},
+					},
+				},
+			},
+			expectedSummary: "Core tests succeeded. Extended tests partially succeeded with 1 test skips.",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tc := tc
+			summary := buildSummary(tc.report)
+			require.Equal(t, tc.expectedSummary, summary)
+		})
+	}
+}