Merge pull request kubernetes#76197 from liggitt/generated-vendor

Mark staging go module files as generated, add script to lint dependencies

Author: k8s-ci-robot

.gitattributes

@@ -7,3 +7,5 @@ test/test_owners.csv merge=union
 **/generated.proto
 **/types_swagger_doc_generated.go linguist-generated=true
 api/openapi-spec/*.json linguist-generated=true
+staging/**/go.mod linguist-generated=true
+staging/**/go.sum linguist-generated=true

hack/lint-dependencies.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Explicitly opt into go modules, even though we're inside a GOPATH directory
+export GO111MODULE=on
+# Explicitly clear GOPATH, to ensure nothing this script calls makes use of that path info
+export GOPATH=
+# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+export GOFLAGS=
+# Detect problematic GOPROXY settings that prevent lookup of dependencies
+if [[ "${GOPROXY:-}" == "off" ]]; then
+  kube::log::error "Cannot run with \$GOPROXY=off"
+  exit 1
+fi
+
+kube::golang::verify_go_version
+kube::util::require-jq
+
+outdated=$(go list -m -json all | jq -r '
+  select(.Replace.Version != null) | 
+  select(.Version != .Replace.Version) | 
+  "\(.Path)
+    pinned:    \(.Replace.Version)
+    preferred: \(.Version)
+    hack/pin-dependency.sh \(.Path) \(.Version)"
+')
+if [[ -n "${outdated}" ]]; then
+  echo "These modules are pinned to versions different than the minimal preferred version."
+  echo "That means that without require directives, a different version would be selected."
+  echo "The command to switch to the minimal preferred version is listed for each module."
+  echo ""
+  echo "${outdated}"
+fi
+
+unused=$(comm -23 \
+  <(go mod edit -json | jq -r '.Replace[] | select(.New.Version != null) | .Old.Path' | sort) \
+  <(go list -m -json all | jq -r .Path | sort))
+if [[ -n "${unused}" ]]; then
+  echo ""
+  echo "Pinned module versions that aren't actually used:"
+  echo "${unused}"
+fi
+
+if [[ -n "${unused}${outdated}" ]]; then
+  exit 1
+fi
+
+echo "All pinned dependencies match their preferred version."
+exit 0

hack/pin-dependency.sh

@@ -35,7 +35,7 @@ export GOPATH=
 export GOFLAGS=
 # Detect problematic GOPROXY settings that prevent lookup of dependencies
 if [[ "${GOPROXY:-}" == "off" ]]; then
-  kube::log::error "Cannot run hack/pin-dependency.sh with \$GOPROXY=off"
+  kube::log::error "Cannot run with \$GOPROXY=off"
   exit 1
 fi
 
@@ -54,12 +54,37 @@ if [[ -z "${dep}" || -z "${sha}" ]]; then
   exit 1
 fi
 
+_tmp="${KUBE_ROOT}/_tmp"
+cleanup() {
+  rm -rf "${_tmp}"
+}
+trap "cleanup" EXIT SIGINT
+cleanup
+mkdir -p "${_tmp}"
+
 # Add the require directive
 echo "Running: go get ${dep}@${sha}"
 go get -d "${dep}@${sha}"
 
 # Find the resolved version
 rev=$(go mod edit -json | jq -r ".Require[] | select(.Path == \"${dep}\") | .Version")
+
+# No entry in go.mod, we must be using the natural version indirectly
+if [[ -z "${rev}" ]]; then
+  # backup the go.mod file, since go list modifies it
+  cp go.mod "${_tmp}/go.mod.bak"
+  # find the revision
+  rev=$(go list -m -json "${dep}" | jq -r .Version)
+  # restore the go.mod file
+  mv "${_tmp}/go.mod.bak" go.mod
+fi
+
+# No entry found
+if [[ -z "${rev}" ]]; then
+  echo "Could not resolve ${sha}"
+  exit 1
+fi
+
 echo "Resolved to ${dep}@${rev}"
 
 # Add the replace directive

hack/update-vendor.sh

@@ -240,6 +240,14 @@ for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
 $(go mod why ${loopback_deps})"
       exit 1
     fi
+
+    # prune unused pinned replace directives
+    comm -23 \
+      <(go mod edit -json | jq -r '.Replace[] | select(.Old.Path == .New.Path) | select(.New.Version != null) | .Old.Path' | sort) \
+      <(go list -m -json all | jq -r .Path | sort) |
+    xargs -L 1 -I {} echo "-dropreplace={}" |
+    xargs -L 100 go mod edit -fmt
+
   popd >/dev/null 2>&1
 done
 echo "=== tidying root" >> "${LOG_FILE}"