Update the conformance list and doc generation logic

The existing walk.go and conformance.txt have a few shortcomings
which we'd like to resolve:
 - difficult to get the full test name due to test context nesting
 - complicated AST logic and understanding necessary due to the
different ways a test can be invoked and written

This changes the AST parsing logic to be much more simple and simply
looks for the comments at/around a specific line. This file/line
information (and the full test name) is gathered by a custom ginkgo
reporter which dumps the SpecSummary data to a file.

Also, the SpecSummary dump can, itself, be potentially useful for
other post-processing and debugging tasks.

Signed-off-by: John Schnake <[email protected]>

Author: johnSchnake

test/conformance/BUILD

@@ -2,9 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["walk.go"],
+    srcs = [
+        "doc.go",
+        "walk.go",
+    ],
     importpath = "k8s.io/kubernetes/test/conformance",
     visibility = ["//visibility:private"],
+    deps = [
+        "//vendor/github.com/onsi/ginkgo/types:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
+    ],
 )
 
 go_binary(
@@ -32,22 +39,35 @@ filegroup(
 )
 
 genrule(
-    name = "list_conformance_tests",
+    name = "list_conformance_specs",
     srcs = [
         "//test/e2e:all-srcs",
         "//test/e2e_node:all-srcs",
+        "//test/e2e:e2e.test_binary",
+        "//vendor/github.com/onsi/ginkgo/ginkgo",
     ],
-    outs = ["conformance.txt"],
-    cmd = "./$(location :conformance) $(locations //test/e2e:all-srcs) > $@",
+    outs = ["specsummaries.json"],
+    cmd = "$(location //vendor/github.com/onsi/ginkgo/ginkgo) --dryRun=true --focus=[Conformance] $(location //test/e2e:e2e.test_binary) -- --spec-dump $$(pwd)/$@ > /dev/null",
+    message = "Getting all conformance spec summaries.",
+)
+
+genrule(
+    name = "list_conformance_tests",
+    srcs = [
+        ":list_conformance_specs",
+        ":conformance",
+        "//test/e2e:all-srcs",
+    ],
+    outs = ["conformance.yaml"],
+    cmd = "$(location :conformance) $(location :list_conformance_specs) > $@",
     message = "Listing all conformance tests.",
-    tools = [":conformance"],
 )
 
 sh_test(
     name = "conformance_test",
     srcs = ["conformance_test.sh"],
     data = [
-        "testdata/conformance.txt",
+        "testdata/conformance.yaml",
         ":list_conformance_tests",
     ],
 )
@@ -58,3 +78,16 @@ go_test(
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
 )
+
+genrule(
+    name = "gen_conformance_docs",
+    srcs = [
+        ":list_conformance_specs",
+        ":conformance",
+        "//test/e2e:all-srcs",
+        ":package-srcs",
+    ],
+    outs = ["conformance.md"],
+    cmd = "$(location :conformance) --docs $(location :list_conformance_specs) > $@",
+    message = "Listing all conformance tests.",
+)

test/conformance/README.md

@@ -9,7 +9,7 @@ To update the list, run
 
 ```console
 bazel build //test/conformance:list_conformance_tests
-cp bazel-genfiles/test/conformance/conformance.txt test/conformance/testdata
+cp bazel-genfiles/test/conformance/conformance.yaml test/conformance/testdata
 ```
 
 Add the changed file to your PR, then send for review.

test/conformance/cf_header.md

@@ -15,26 +15,24 @@ clean up the tests.
 Example:
 ```
 /*
-  Testname: Kubelet-OutputToLogs
-  Release: v1.9
-  Description: By default the stdout and stderr from the process
-           being executed in a pod MUST be sent to the pod's logs.
+  Release : v1.13
+  Testname: Kubelet, log output, default
+  Description: By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
 */
-// Note this test needs to be fixed to also test for stderr
-It("it should print the output to logs [Conformance]", func() {
+framework.ConformanceIt("should print the output to logs [NodeConformance]", func() {
 ```
 
 would generate the following documentation for the test. Note that the "TestName" from the Documentation above will
 be used to document the test which make it more human readable. The "Description" field will be used as the
 documentation for that test.
 
 ### **Output:**
-## [Kubelet-OutputToLogs](https://github.com/kubernetes/kubernetes/blob/release-1.9/test/e2e_node/kubelet_test.go#L42)
+## [Kubelet, log output, default](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/common/kubelet.go#L48)
 
-### Release v1.9
-By default the stdout and stderr from the process
-being executed in a pod MUST be sent to the pod's logs.
-Note this test needs to be fixed to also test for stderr
+- Added to conformance in release v1.13
+- Defined in code as: [k8s.io] Kubelet when scheduling a busybox command in a pod should print the output to logs [NodeConformance] [Conformance]
+
+By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
 
 Notational Conventions when documenting the tests with the key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119).
 

test/conformance/conformance_test.sh

@@ -21,7 +21,7 @@
 
 set -o errexit
 
-if diff -u test/conformance/testdata/conformance.txt test/conformance/conformance.txt; then
+if diff -u test/conformance/testdata/conformance.yaml test/conformance/conformance.yaml; then
   echo PASS
   exit 0
 fi

test/conformance/doc.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+This stand-alone package is utilized for dynamically generating/maintaining a list of
+conformance tests. It utilizes a two step approach:
+ - The test binary is built
+ - The test binary is run in dry mode with a custom ginkgo reporter dumping out
+ types.SpecSummary objects which contain full test names and file/code information.
+ - The SpecSummary information is parsed to get file/line info on Conformance tests and
+ then we use a simplified AST parser to grab the comments above the test.
+
+Due to the complicated nature of how tests can be declared/wrapped in various contexts,
+this approach is much simpler to maintain than a pure-AST parser and allows us to easily
+capture the full test names/locations of the tests using the pre-existing ginkgo logic.
+*/
+package main