Merge pull request kubernetes#76586 from JishanXing/76093

Fix kubernetes#76093, restructure LICENSES file generation

Author: k8s-ci-robot

Godeps/OWNERS

@@ -729,8 +729,6 @@ function kube::build::copy_output() {
     --prune-empty-dirs \
     --filter='- /_temp/' \
     --filter='+ /vendor/' \
-    --filter='+ /Godeps/' \
-    --filter='+ /staging/***/Godeps/**' \
     --filter='+ /_output/dockerized/bin/**' \
     --filter='+ zz_generated.*' \
     --filter='+ generated.proto' \

build/common.sh

@@ -187,7 +187,8 @@ function kube::release::package_node_tarballs() {
     cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
       "${release_stage}/node/bin/"
 
-    cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
+    cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/"
+    cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES"
 
     cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"
 
@@ -252,7 +253,8 @@ function kube::release::package_server_tarballs() {
     cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
       "${release_stage}/server/bin/"
 
-    cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
+    cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/"
+    cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES"
 
     cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"
 
@@ -565,7 +567,8 @@ EOF
 
   cp -R "${KUBE_ROOT}/docs" "${release_stage}/"
   cp "${KUBE_ROOT}/README.md" "${release_stage}/"
-  cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
+  cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/"
+  cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES"
 
   echo "${KUBE_GIT_VERSION}" > "${release_stage}/version"
 

build/lib/release.sh

@@ -69,12 +69,21 @@ pkg_tar(
     ],
 )
 
+filegroup(
+    name = "all-licenses-files",
+    srcs = glob([
+        "vendor/ALL_LICENSES/*",
+    ]),
+    visibility = ["//visibility:private"],
+)
+
 # Included in node and server tarballs.
 filegroup(
     name = "license-targets",
     srcs = [
+        ":all-licenses-files",
         ":kubernetes-src.tar.gz",
-        "//:Godeps/LICENSES",
+        "//:LICENSE",
     ],
     visibility = ["//visibility:private"],
 )
@@ -235,7 +244,8 @@ pkg_tar(
 pkg_tar(
     name = "kubernetes",
     srcs = [
-        "//:Godeps/LICENSES",
+        ":all-licenses-files",
+        "//:LICENSE",
         "//:README.md",
         "//:version",
         "//cluster:all-srcs",

build/release-tars/BUILD

@@ -421,7 +421,7 @@ function install-kube-binary-config {
     mv "${src_dir}/kubelet" "${KUBE_BIN}"
     mv "${src_dir}/kubectl" "${KUBE_BIN}"
 
-    mv "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}"
+    mv "${KUBE_HOME}/kubernetes/LICENSE" "${KUBE_HOME}"
     mv "${KUBE_HOME}/kubernetes/kubernetes-src.tar.gz" "${KUBE_HOME}"
   fi
 

cluster/gce/gci/configure.sh

@@ -155,7 +155,7 @@ def file_extension(filename):
     return os.path.splitext(filename)[1].split(".")[-1].lower()
 
 
-skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
+skipped_dirs = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
                 "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test",
                 "staging/src/k8s.io/kubectl/pkg/generated/bindata.go"]
 

hack/boilerplate/boilerplate.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Update the Godeps/LICENSES document.
+# Update the vendor/ALL_LICENSES document.
 # Generates a table of Godep dependencies and their license.
 #
 # Usage:
@@ -24,12 +24,15 @@
 #    additionally created files into the vendor auto-generated tree.
 #
 #    Run every time a license file is added/modified within /vendor to
-#    update /Godeps/LICENSES
+#    update /vendor/ALL_LICENSES
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
 export LANG=C
 export LC_ALL=C
 
@@ -130,8 +133,6 @@ process_content () {
 #############################################################################
 # MAIN
 #############################################################################
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
 
 export GO111MODULE=on
 
@@ -153,8 +154,19 @@ fi
 LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}"
 cd "${LICENSE_ROOT}"
 
-VENDOR_LICENSE_FILE="Godeps/LICENSES"
-TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$"
+kube::util::ensure-temp-dir
+
+# Save the genreated LICENSE file for each package temporarily
+TMP_LICENSE_FILE="${KUBE_TEMP}/vendor.LICENSES.$$"
+
+# The directory to save all the LICENSE files
+VENDOR_LICENSE_DIR="${LICENSE_ROOT}/vendor/ALL_LICENSES"
+
+# The tmp directory to save all the LICENSE files, will move to VENDOR_LICENSE_DIR
+TMP_VENDOR_LICENSE_DIR="${KUBE_TEMP}/vendor.LICENSES.DIR.$$"
+
+mkdir -p "${TMP_VENDOR_LICENSE_DIR}"
+
 DEPS_DIR="vendor"
 declare -Ag CONTENT
 
@@ -167,7 +179,8 @@ cat "${LICENSE_ROOT}/LICENSE"
 echo
 echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")"
 echo "================================================================================"
-) > ${TMP_LICENSE_FILE}
+) > "${TMP_LICENSE_FILE}"
+mv "${TMP_LICENSE_FILE}" "${TMP_VENDOR_LICENSE_DIR}/LICENSE"
 
 # Loop through every vendored package
 for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
@@ -184,11 +197,10 @@ for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
   process_content "${PACKAGE}" COPYRIGHT
   process_content "${PACKAGE}" COPYING
 
-  # display content
-  echo
-  echo "================================================================================"
-  echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
-  echo
+  # copy content and throw error message
+  {
+    echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
+	echo
 
   file=""
   if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
@@ -211,12 +223,20 @@ Options:
 __EOF__
       exit 9
   fi
-  cat "${file}"
+  
+    cat "${file}"
+    echo
+    echo "= ${file} $(kube::util::md5 "${file}")"
+  } >> "${TMP_LICENSE_FILE}"
 
-  echo
-  echo "= ${file} $(kube::util::md5 "${file}")"
-  echo "================================================================================"
-  echo
-done >> ${TMP_LICENSE_FILE}
 
-cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE}
+  dest_dir="${TMP_VENDOR_LICENSE_DIR}/${PACKAGE}"
+  mkdir -p "${dest_dir}"
+  mv "${TMP_LICENSE_FILE}" "${dest_dir}/LICENSE"
+done
+
+if [[ -d "${VENDOR_LICENSE_DIR}" ]]; then
+  rm -r "${VENDOR_LICENSE_DIR}"
+fi
+mv "${TMP_VENDOR_LICENSE_DIR}" "${VENDOR_LICENSE_DIR}"
+

hack/update-vendor-licenses.sh

@@ -340,11 +340,10 @@ kube::log::status "vendor: updating LICENSES file"
 hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1
 
 kube::log::status "vendor: creating OWNERS file"
-rm -f "Godeps/OWNERS" "vendor/OWNERS"
-cat <<__EOF__ > "Godeps/OWNERS"
+rm -f "vendor/OWNERS"
+cat <<__EOF__ > "vendor/OWNERS"
 # See the OWNERS docs at https://go.k8s.io/owners
 
 approvers:
 - dep-approvers
 __EOF__
-cp "Godeps/OWNERS" "vendor/OWNERS"

hack/update-vendor.sh

@@ -33,16 +33,25 @@ function cleanup {
 }
 kube::util::trap_add cleanup EXIT
 
-cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps"
+#ln -s all subfolder except ALL_LICENSES in vendor
+mkdir -p "${_tmpdir}/vendor"
+for child in "${KUBE_ROOT}/vendor"/*
+do
+  if [[ "${child}" == "${KUBE_ROOT}/vendor/ALL_LICENSES" ]] ; then
+    continue
+  else
+     ln -s "${child}" "${_tmpdir}/vendor"
+  fi
+done
+
 ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}"
-ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}"
 ln -s "${KUBE_ROOT}/staging" "${_tmpdir}"
 
 # Update vendor Licenses
 LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh"
 
 # Compare vendor Licenses
-if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then
+if ! _out="$(diff -Naupr "${KUBE_ROOT}/vendor/ALL_LICENSES" "${_tmpdir}/vendor/ALL_LICENSES")"; then
   echo "Your vendor licenses file is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2
   echo "${_out}" >&2
   exit 1

hack/verify-vendor-licenses.sh

@@ -55,7 +55,7 @@ export GO111MODULE=on
 
 pushd "${_kubetmp}" > /dev/null 2>&1
   # Destroy deps in the copy of the kube tree
-  rm -rf ./Godeps/LICENSES ./vendor
+  rm -rf ./vendor
 
   # Recreate the vendor tree using the nice clean set we just downloaded
   hack/update-vendor.sh