Merge pull request kubernetes#127606 from thockin/skip_test_target_normalization

Skip Go target normalization in integration tests

Author: k8s-ci-robot

hack/install-etcd.sh

@@ -25,4 +25,6 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
+FOUND="$(echo "${PATH}" | sed 's/:/\n/g' | grep -q "^${KUBE_ROOT}/third_party/etcd$" || true)"
 kube::etcd::install
+test -n "${FOUND}" || echo "  PATH=\"\$PATH:${KUBE_ROOT}/third_party/etcd\""

hack/lib/etcd.sh

@@ -159,7 +159,7 @@ kube::etcd::install() {
 
     cd "${KUBE_ROOT}/third_party" || return 1
     if [[ $(readlink etcd) == etcd-v${ETCD_VERSION}-${os}-* ]]; then
-      V=3 kube::log::info "etcd v${ETCD_VERSION} is already installed"
+      V=4 kube::log::info "etcd v${ETCD_VERSION} is already installed"
       return 0 # already installed
     fi
 

hack/lib/golang.sh

@@ -383,7 +383,7 @@ kube::golang::best_guess_go_targets() {
       continue
     fi
 
-    if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/" ]]; then
+    if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/".+ ]]; then
       # If the target starts with what looks like a domain name, assume it has a
       # fully-qualified Go package name.
       echo "${target}"
@@ -413,6 +413,19 @@ kube::golang::best_guess_go_targets() {
   done
 }
 
+kube::golang::internal::lazy_normalize() {
+  target="$1"
+
+  if [[ "${target}" =~ ^([[:alnum:]]+".")+[[:alnum:]]+"/".+ ]]; then
+    # If the target starts with what looks like a domain name, assume it has a
+    # fully-qualified Go package name.
+    echo "${target}"
+    return
+  fi
+
+  go list -find -e "${target}"
+}
+
 # kube::golang::normalize_go_targets takes a list of build targets, which might
 # be Go-style names (e.g. example.com/foo/bar or ./foo/bar) or just local paths
 # (e.g. foo/bar) and produces a respective list (on stdout) of Go package
@@ -433,19 +446,19 @@ kube::golang::normalize_go_targets() {
       local tst
       tst="$(basename "${target}")"
       local pkg
-      pkg="$(go list -find -e "${dir}")"
+      pkg="$(kube::golang::internal::lazy_normalize "${dir}")"
       echo "${pkg}/${tst}"
       continue
     fi
     if [[ "${target}" =~ "/..."$ ]]; then
       local dir
       dir="$(dirname "${target}")"
       local pkg
-      pkg="$(go list -find -e "${dir}")"
+      pkg="$(kube::golang::internal::lazy_normalize "${dir}")"
       echo "${pkg}/..."
       continue
     fi
-    go list -find -e "${target}"
+    kube::golang::internal::lazy_normalize "${target}"
   done
 }
 

hack/make-rules/test-integration.sh

@@ -21,6 +21,10 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
+kube::golang::setup_env
+kube::golang::setup_gomaxprocs
+kube::util::require-jq
+
 # start the cache mutation detector by default so that cache mutators will be found
 KUBE_CACHE_MUTATION_DETECTOR="${KUBE_CACHE_MUTATION_DETECTOR:-true}"
 export KUBE_CACHE_MUTATION_DETECTOR
@@ -43,16 +47,23 @@ KUBE_TEST_ARGS=${KUBE_TEST_ARGS:-}
 # Default glog module settings.
 KUBE_TEST_VMODULE=${KUBE_TEST_VMODULE:-""}
 
-kube::test::find_integration_test_dirs() {
+kube::test::find_integration_test_pkgs() {
   (
     cd "${KUBE_ROOT}"
-    # The "./" syntax here produces Go-compatible package names.
-    find ./test/integration/ -name '*_test.go' -print0 \
-      | xargs -0n1 dirname \
-      | LC_ALL=C sort -u
-    find ./staging/src/k8s.io/apiextensions-apiserver/test/integration/ -name '*_test.go' -print0 \
-      | xargs -0n1 dirname \
-      | LC_ALL=C sort -u
+
+    # Get a list of all the modules in this workspace.
+    local -a workspace_module_patterns
+    kube::util::read-array workspace_module_patterns < <(
+        go list -m -json | jq -r '.Dir' \
+        | while read -r D; do
+            SUB="${D}/test/integration";
+            test -d "${SUB}" && echo "${SUB}/...";
+        done)
+
+    # Get a list of all packages which have test files.
+    go list -find \
+        -f '{{if or (gt (len .TestGoFiles) 0) (gt (len .XTestGoFiles) 0)}}{{.ImportPath}}{{end}}' \
+        "${workspace_module_patterns[@]}"
   )
 }
 
@@ -81,7 +92,7 @@ runTests() {
   # empty here to ensure that we aren't unintentionally consuming them from the
   # previous make invocation.
   KUBE_TEST_ARGS="${SHORT:--short=true} --vmodule=${KUBE_TEST_VMODULE} ${KUBE_TEST_ARGS}" \
-      WHAT="${WHAT:-$(kube::test::find_integration_test_dirs | paste -sd' ' -)}" \
+      WHAT="${WHAT:-$(kube::test::find_integration_test_pkgs | paste -sd' ' -)}" \
       GOFLAGS="${GOFLAGS:-}" \
       KUBE_TIMEOUT="${KUBE_TIMEOUT}" \
       KUBE_RACE="" \

hack/make-rules/test.sh

@@ -158,7 +158,7 @@ if [[ ${#testcases[@]} -eq 0 ]]; then
   kube::util::read-array testcases < <(kube::test::find_go_packages)
 else
   # If the user passed targets, we should normalize them.
-  # This can be slow!
+  # This can be slow for large numbers of inputs.
   kube::log::status "Normalizing Go targets"
   kube::util::read-array testcases < <(kube::golang::normalize_go_targets "${testcases[@]}")
 fi