Merge pull request #142 from tssurya/implement-conformance-profiles

Add Conformance Profiles Test Reporting System

Author: k8s-ci-robot

Makefile

@@ -53,7 +53,15 @@ crd-e2e:
 
 .PHONY: conformance
 conformance:
-	go test -v ./conformance/... -args ${CONFORMANCE_FLAGS}
+	go test ${GO_TEST_FLAGS} -v ./conformance -run TestConformance -args ${CONFORMANCE_FLAGS}
+
+.PHONY: conformance-profiles
+conformance-profiles:
+	go test ${GO_TEST_FLAGS} -v ./conformance -run TestConformanceProfiles -args ${CONFORMANCE_FLAGS}
+
+.PHONY: conformance-profiles-default
+conformance-profiles-default:
+	go test ${GO_TEST_FLAGS} -v ./conformance -run TestConformanceProfiles -args --conformance-profiles=AdminNetworkPolicy,BaselineAdminNetworkPolicy
 
 ##@ Deployment
 install: generate ## Install CRDs into the K8s cluster specified in ~/.kube/config.

conformance/apis/v1alpha1/conformancereport.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ConformanceReport is a report of conformance testing results including the
+// specific conformance profiles that were tested and the results of the tests
+// with summaries and statistics.
+type ConformanceReport struct {
+	metav1.TypeMeta `json:",inline"`
+	Implementation  `json:"implementation"`
+
+	// Date indicates the date that this report was generated.
+	Date string `json:"date"`
+
+	// NetworkPolicyV2APIVersion indicates which release version of NetworkPolicy API this
+	// test report was made for.
+	NetworkPolicyV2APIVersion string `json:"networkPolicyV2APIVersion"`
+
+	// ProfileReports is a list of the individual reports for each conformance
+	// profile that was enabled for a test run.
+	ProfileReports []ProfileReport `json:"profiles"`
+}
+
+// Implementation provides metadata information on the downstream
+// implementation of Network Policy V2 API which ran conformance tests.
+type Implementation struct {
+	// Organization refers to the company, group or individual which maintains
+	// the named implementation. Organizations can provide reports for any
+	// number of distinct Network Policy API implementations they maintain, but need
+	// to identify themselves using this organization field for grouping.
+	Organization string `json:"organization"`
+
+	// Project indicates the name of the project or repository for a Network Policy API
+	// implementation.
+	Project string `json:"project"`
+
+	// URL indicates a human-usable URL where more information about the
+	// implementation can be found. For open source projects this should
+	// generally link to the code repository.
+	URL string `json:"url"`
+
+	// Version indicates the version of the implementation that was used for
+	// testing. This should generally be a semver version when applicable.
+	Version string `json:"version"`
+
+	// Contact is contact information for the maintainers so that Network Policy API
+	// maintainers can get ahold of them as needed. Ideally this should be
+	// Github usernames (in the form of `@<username>`) or team names (in the
+	// form of `@<team>/<name>`), but when that's not possible it can be email
+	// addresses.
+	// Rather than Github usernames or email addresses you can provide a URL to the relevant
+	// support pages for the project. Ideally this would be something like the issue creation page
+	// on a repository, but for projects without a publicly exposed repository a general support
+	// page URL can be provided.
+	Contact []string `json:"contact"`
+
+	// AdditionalInformation field must be used by implementations to provide a link to
+	// the implementation of the APIs OR github actions OR jenkins or any other CI/CD job
+	// definition that shows proof of what was used to generate the report. This will help
+	// maintainers make an informed decision on merging reports into the NetworkPolicyAPI project.
+	AdditionalInformation string `json:"additionalInformation"`
+}

conformance/apis/v1alpha1/doc.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// V1alpha1 includes alpha maturity API types and utilities for creating and
+// handling the results of conformance test runs. These types are _only_
+// intended for use by the conformance test suite OR external test suites that
+// are written in Golang and execute the conformance test suite as a Golang
+// library.
+//
+// Note that currently all sub-packages are not intended for general use or
+// to be distributed as part of a release so there is no way to use them by
+// default when using the Golang library at this time. If you don't know for
+// sure that you want to use these features, then you should not use them.
+// If you would like to opt into these unreleased features use Go build tags
+// to enable them, e.g.:
+//
+//   $ GOFLAGS='-tags=experimental' go test ./conformance/... -args ${CONFORMANCE_ARGS}
+//
+// Please note that everything here is considered experimental and subject to
+// change. Expect breaking changes and/or complete removals if you start using
+// them.
+
+package v1alpha1

conformance/apis/v1alpha1/profilereport.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+// ProfileReport is the generated report for the test results of a specific
+// named conformance profile.
+type ProfileReport struct {
+	// Name indicates the name of the conformance profile
+	// (e.g. "AdminNetworkPolicy", "BaselineAdminNetworkPolicy")
+	Name string `json:"name"`
+
+	// Core indicates the core support level which includes the set of tests
+	// which are the minimum the implementation must pass to be considered at
+	// all conformant.
+	Core Status `json:"core"`
+
+	// Extended indicates the extended support level which includes additional
+	// optional features which the implementation may choose to implement
+	// support for, but are not required.
+	Extended *ExtendedStatus `json:"extended,omitempty"`
+}
+
+// ExtendedStatus shows the testing results for the extended support level.
+type ExtendedStatus struct {
+	Status `json:",inline"`
+
+	// SupportedFeatures indicates which extended features were flagged as
+	// supported by the implementation and tests will be attempted for.
+	SupportedFeatures []string `json:"supportedFeatures,omitempty"`
+
+	// UnsupportedFeatures indicates which extended features the implementation
+	// does not have support for and therefore will not attempt to test.
+	UnsupportedFeatures []string `json:"unsupportedFeatures,omitempty"`
+}
+
+// Status includes details on the results of a test.
+type Status struct {
+	Result `json:"result"`
+
+	// Summary is a human-readable message intended for end-users to understand
+	// the overall status at a glance.
+	Summary string `json:"summary"`
+
+	// Statistics includes numerical statistics on the result of the test run.
+	Statistics `json:"statistics"`
+
+	// SkippedTests indicates which tests were explicitly disabled in the test
+	// suite. Skipping tests for Core level support implicitly identifies the
+	// results as being partial and the implementation will not be considered
+	// conformant at any level.
+	SkippedTests []string `json:"skippedTests,omitempty"`
+
+	// FailedTests indicates which tests were failing during the execution of
+	// test suite.
+	FailedTests []string `json:"failedTests,omitempty"`
+}

conformance/apis/v1alpha1/result.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+// Result is a simple high-level summary describing the conclusion of a test
+// run.
+type Result string
+
+var (
+	// Success indicates that the test run concluded in all required tests
+	// passing.
+	Success Result = "success"
+
+	// Partial indicates that the test run concluded in some of the required
+	// tests passing without any failures, but some were skipped.
+	Partial Result = "partial"
+
+	// Failure indicates that the test run concluded in one ore more tests
+	// failing to complete successfully.
+	Failure Result = "failure"
+)

conformance/apis/v1alpha1/statistics.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+// Statistics includes numerical summaries of the number of conformance tests
+// that passed, failed or were intentionally skipped.
+type Statistics struct {
+	// Passed indicates how many tests completed successfully.
+	Passed uint32
+
+	// Skipped indicates how many tests were intentionally not run, whether due
+	// to lack of feature support or whether they were explicitly disabled in
+	// the test suite.
+	Skipped uint32
+
+	// Failed indicates how many tests were unsuccessful.
+	Failed uint32
+}