Merge pull request kubernetes#88486 from Jefftree/schnake-pr

Adds custom reporter and logic to generate conformance docs and list of tests

Author: k8s-ci-robot

test/conformance/BUILD

@@ -2,9 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["walk.go"],
+    srcs = [
+        "doc.go",
+        "walk.go",
+    ],
     importpath = "k8s.io/kubernetes/test/conformance",
     visibility = ["//visibility:private"],
+    deps = [
+        "//vendor/github.com/onsi/ginkgo/types:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
+    ],
 )
 
 go_binary(
@@ -32,22 +39,35 @@ filegroup(
 )
 
 genrule(
-    name = "list_conformance_tests",
+    name = "list_conformance_specs",
     srcs = [
         "//test/e2e:all-srcs",
         "//test/e2e_node:all-srcs",
+        "//test/e2e:e2e.test_binary",
+        "//vendor/github.com/onsi/ginkgo/ginkgo",
     ],
-    outs = ["conformance.txt"],
-    cmd = "./$(location :conformance) $(locations //test/e2e:all-srcs) > $@",
+    outs = ["specsummaries.json"],
+    cmd = "$(location //vendor/github.com/onsi/ginkgo/ginkgo) --dryRun=true --focus=[Conformance] $(location //test/e2e:e2e.test_binary) -- --spec-dump $$(pwd)/$@ > /dev/null",
+    message = "Getting all conformance spec summaries.",
+)
+
+genrule(
+    name = "list_conformance_tests",
+    srcs = [
+        ":list_conformance_specs",
+        ":conformance",
+        "//test/e2e:all-srcs",
+    ],
+    outs = ["conformance.yaml"],
+    cmd = "$(location :conformance) $(location :list_conformance_specs) > $@",
     message = "Listing all conformance tests.",
-    tools = [":conformance"],
 )
 
 sh_test(
     name = "conformance_test",
     srcs = ["conformance_test.sh"],
     data = [
-        "testdata/conformance.txt",
+        "testdata/conformance.yaml",
         ":list_conformance_tests",
     ],
 )
@@ -58,3 +78,16 @@ go_test(
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
 )
+
+genrule(
+    name = "gen_conformance_docs",
+    srcs = [
+        ":list_conformance_specs",
+        ":conformance",
+        "//test/e2e:all-srcs",
+        ":package-srcs",
+    ],
+    outs = ["conformance.md"],
+    cmd = "$(location :conformance) --docs $(location :list_conformance_specs) > $@",
+    message = "Listing all conformance tests.",
+)

test/conformance/README.md

@@ -9,7 +9,7 @@ To update the list, run
 
 ```console
 bazel build //test/conformance:list_conformance_tests
-cp bazel-genfiles/test/conformance/conformance.txt test/conformance/testdata
+cp bazel-genfiles/test/conformance/conformance.yaml test/conformance/testdata
 ```
 
 Add the changed file to your PR, then send for review.

test/conformance/cf_header.md

@@ -15,26 +15,24 @@ clean up the tests.
 Example:
 ```
 /*
-  Testname: Kubelet-OutputToLogs
-  Release: v1.9
-  Description: By default the stdout and stderr from the process
-           being executed in a pod MUST be sent to the pod's logs.
+  Release : v1.13
+  Testname: Kubelet, log output, default
+  Description: By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
 */
-// Note this test needs to be fixed to also test for stderr
-It("it should print the output to logs [Conformance]", func() {
+framework.ConformanceIt("should print the output to logs [NodeConformance]", func() {
 ```
 
 would generate the following documentation for the test. Note that the "TestName" from the Documentation above will
 be used to document the test which make it more human readable. The "Description" field will be used as the
 documentation for that test.
 
 ### **Output:**
-## [Kubelet-OutputToLogs](https://github.com/kubernetes/kubernetes/blob/release-1.9/test/e2e_node/kubelet_test.go#L42)
+## [Kubelet, log output, default](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/common/kubelet.go#L48)
 
-### Release v1.9
-By default the stdout and stderr from the process
-being executed in a pod MUST be sent to the pod's logs.
-Note this test needs to be fixed to also test for stderr
+- Added to conformance in release v1.13
+- Defined in code as: [k8s.io] Kubelet when scheduling a busybox command in a pod should print the output to logs [NodeConformance] [Conformance]
+
+By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
 
 Notational Conventions when documenting the tests with the key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119).
 

test/conformance/conformance_test.sh

@@ -21,7 +21,7 @@
 
 set -o errexit
 
-if diff -u test/conformance/testdata/conformance.txt test/conformance/conformance.txt; then
+if diff -u test/conformance/testdata/conformance.yaml test/conformance/conformance.yaml; then
   echo PASS
   exit 0
 fi

test/conformance/doc.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+This stand-alone package is utilized for dynamically generating/maintaining a list of
+conformance tests. It utilizes a two step approach:
+ - The test binary is built
+ - The test binary is run in dry mode with a custom ginkgo reporter dumping out
+ types.SpecSummary objects which contain full test names and file/code information.
+ - The SpecSummary information is parsed to get file/line info on Conformance tests and
+ then we use a simplified AST parser to grab the comments above the test.
+
+Due to the complicated nature of how tests can be declared/wrapped in various contexts,
+this approach is much simpler to maintain than a pure-AST parser and allows us to easily
+capture the full test names/locations of the tests using the pre-existing ginkgo logic.
+*/
+package main