Add support for listing conformance tests

we print yaml, so you can use yaml tools like `yq`:
```
e2e.test --list-conformance-tests | yq r - --collect *.testname
```

Signed-off-by: Davanum Srinivas <[email protected]>

Author: dims

test/conformance/BUILD

@@ -34,6 +34,7 @@ filegroup(
         ":package-srcs",
         "//test/conformance/behaviors:all-srcs",
         "//test/conformance/kubeconform:all-srcs",
+        "//test/conformance/testdata:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
@@ -68,8 +69,8 @@ sh_test(
     name = "conformance_test",
     srcs = ["conformance_test.sh"],
     data = [
-        "testdata/conformance.yaml",
         ":list_conformance_tests",
+        ":testdata",
     ],
 )
 

test/conformance/testdata/BUILD

@@ -0,0 +1,14 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/e2e/BUILD

@@ -43,6 +43,7 @@ go_test(
         "//test/e2e/windows:go_default_library",
         "//test/utils/image:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
     ],
 )
 

test/e2e/e2e_test.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	// Never, ever remove the line with "/ginkgo". Without it,
 	// the ginkgo test runner will not detect that this
 	// directory contains a Ginkgo test suite.
@@ -94,6 +96,32 @@ func TestMain(m *testing.M) {
 		os.Exit(0)
 	}
 
+	// Enable bindata file lookup as fallback.
+	testfiles.AddFileSource(testfiles.BindataFileSource{
+		Asset:      generated.Asset,
+		AssetNames: generated.AssetNames,
+	})
+	if framework.TestContext.ListConformanceTests {
+		var tests []struct {
+			Testname    string `yaml:"testname"`
+			Codename    string `yaml:"codename"`
+			Description string `yaml:"description"`
+			Release     string `yaml:"release"`
+			File        string `yaml:"file"`
+		}
+
+		data := testfiles.ReadOrDie("test/conformance/testdata/conformance.yaml")
+		if err := yaml.Unmarshal(data, &tests); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		if err := yaml.NewEncoder(os.Stdout).Encode(tests); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
 	framework.AfterReadingAllFlags(&framework.TestContext)
 
 	// TODO: Deprecating repo-root over time... instead just use gobindata_util.go , see #23987.
@@ -105,12 +133,6 @@ func TestMain(m *testing.M) {
 		testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot})
 	}
 
-	// Enable bindata file lookup as fallback.
-	testfiles.AddFileSource(testfiles.BindataFileSource{
-		Asset:      generated.Asset,
-		AssetNames: generated.AssetNames,
-	})
-
 	rand.Seed(time.Now().UnixNano())
 	os.Exit(m.Run())
 }

test/e2e/framework/test_context.go

@@ -83,6 +83,9 @@ type TestContextType struct {
 	// ListImages will list off all images that are used then quit
 	ListImages bool
 
+	// ListConformanceTests will list off all conformance tests that are available then quit
+	ListConformanceTests bool
+
 	// Provider identifies the infrastructure provider (gce, gke, aws)
 	Provider string
 
@@ -301,6 +304,7 @@ func RegisterCommonFlags(flags *flag.FlagSet) {
 	flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/master`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests.")
 
 	flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for runnning tests.")
+	flags.BoolVar(&TestContext.ListConformanceTests, "list-conformance-tests", false, "If true, will show list of conformance tests.")
 	flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
 
 	flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.")