Merge pull request #386 from ggriffiths/r21_fix_build_install_go115

release-2.1: Update release tools for go build fix

Author: k8s-ci-robot

release-tools/SIDECAR_RELEASE_PROCESS.md

@@ -50,7 +50,7 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 ## Release Process
 1. Identify all issues and ongoing PRs that should go into the release, and
   drive them to resolution.
-1. Download [K8s release notes
+1. Download v2.8+ [K8s release notes
   generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
 1. Generate release notes for the release. Replace arguments with the relevant
   information.
@@ -62,13 +62,10 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
         ```
     * For new patch releases on a release branch:
         ```
-        GITHUB_TOKEN=<token> release-notes --branch=release-1.1
-        --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b
+        GITHUB_TOKEN=<token> release-notes --discover=patch-to-latest --branch=release-1.1
         --github-org=kubernetes-csi --github-repo=external-provisioner
         --required-author="" --output out.md
         ```
-        * `--start-rev` should point to the last patch release from the release branch.
-        * `--end-sha` should point to the latest commit from the release branch.
 1. Compare the generated output to the new commits for the release to check if
    any notable change missed a release note.
 1. Reword release notes as needed. Make sure to check notes for breaking
@@ -89,6 +86,12 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
    [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new)
 1. If release was a new major/minor version, create a new `release-<minor>`
    branch at that commit.
+1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
+1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From
+   the [k8s image
+   repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage),
+   run `./generate.sh > images.yaml`, and send a PR with the updated images.
+   Once merged, the image promoter will copy the images from staging to prod.
 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar
    and feature pages with the new released version.
 1. After all the sidecars have been released, update

release-tools/build.make

@@ -71,7 +71,7 @@ BUILD_PLATFORMS =
 
 # This builds each command (= the sub-directories of ./cmd) for the target platform(s)
 # defined by BUILD_PLATFORMS.
-build-%: check-go-version-go
+$(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
 	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
@@ -80,10 +80,10 @@ build-%: check-go-version-go
 		fi; \
 	done
 
-container-%: build-%
+$(CMDS:%=container-%): container-%: build-%
 	docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
 
-push-%: container-%
+$(CMDS:%=push-%): push-%: container-%
 	set -ex; \
 	push_image () { \
 		docker tag $*:latest $(IMAGE_NAME):$$tag; \
@@ -105,6 +105,77 @@ build: $(CMDS:%=build-%)
 container: $(CMDS:%=container-%)
 push: $(CMDS:%=push-%)
 
+# Additional parameters are needed when pushing to a local registry,
+# see https://github.com/docker/buildx/issues/94.
+# However, that then runs into https://github.com/docker/cli/issues/2396.
+#
+# What works for local testing is:
+# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
+DOCKER_BUILDX_CREATE_ARGS ?=
+
+# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
+# Docker Buildx is included in Docker 19.03.
+#
+# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# It is currently optional: if no such file exists, Windows images are not included,
+# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that
+# Windows binaries can be built before adding a Dockerfile for it.
+#
+# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
+# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# the tag for the resulting multiarch image.
+$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
+	set -ex; \
+	DOCKER_CLI_EXPERIMENTAL=enabled; \
+	export DOCKER_CLI_EXPERIMENTAL; \
+	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
+	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
+	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
+	if ! [ -f "$$dockerfile_windows" ]; then \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+	fi; \
+	pushMultiArch () { \
+		tag=$$1; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+			docker buildx build --push \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--platform=$$os/$$arch \
+				--file $$(eval echo \$${dockerfile_$$os}) \
+				--build-arg binary=./bin/$*$$suffix \
+				--label revision=$(REV) \
+				.; \
+		done; \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		docker manifest push -p $(IMAGE_NAME):$$tag; \
+	}; \
+	if [ $(PULL_BASE_REF) = "master" ]; then \
+			: "creating or overwriting canary image"; \
+			pushMultiArch canary; \
+	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
+			: "creating or overwriting canary image for release branch"; \
+			release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
+			pushMultiArch $$release_canary_tag; \
+	elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
+			: "creating release image"; \
+			pushMultiArch $(PULL_BASE_REF); \
+	else \
+			: "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \
+			exit 1; \
+	fi
+
+.PHONY: check-pull-base-ref
+check-pull-base-ref:
+	if ! [ "$(PULL_BASE_REF)" ]; then \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		exit 1; \
+	fi
+
+.PHONY: push-multiarch
+push-multiarch: $(CMDS:%=push-multiarch-%)
+
 clean:
 	-rm -rf bin
 

release-tools/cloudbuild.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# shellcheck disable=SC1091
+. release-tools/prow.sh
+
+gcr_cloud_build

release-tools/cloudbuild.yaml

@@ -0,0 +1,46 @@
+# A configuration file for multi-arch image building with the Google cloud build service.
+#
+# Repos using this file must:
+# - import csi-release-tools
+# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml
+# - add a .cloudbuild.sh which can be a custom file or a symlink
+#   to release-tools/cloudbuild.sh
+# - accept "binary" as build argument in their Dockerfile(s) (see
+#   https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6)
+#   because binaries will get built for different architectures and then
+#   get copied from the built host into the container image
+#
+# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
+# for more details on image pushing process in Kubernetes.
+#
+# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage.
+
+# This must be specified in seconds. If omitted, defaults to 600s (10 mins).
+timeout: 1800s
+# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+steps:
+  # The image must contain bash and curl. Ideally it should also contain
+  # the desired version of Go (currently defined in release-tools/travis.yml),
+  # but that just speeds up the build and is not required.
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+    entrypoint: ./.cloudbuild.sh
+    env:
+    - GIT_TAG=${_GIT_TAG}
+    - PULL_BASE_REF=${_PULL_BASE_REF}
+    - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT}
+    - HOME=/root
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution.
+  _GIT_TAG: '12345'
+  # _PULL_BASE_REF will contain the ref that was pushed to trigger this build -
+  # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
+  _PULL_BASE_REF: 'master'
+  # The default gcr.io staging project for Kubernetes-CSI
+  # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL).
+  # Might be overridden in the Prow build job for a repo which wants
+  # images elsewhere.
+  _STAGING_PROJECT: 'k8s-staging-sig-storage'

release-tools/filter-junit.go

@@ -15,10 +15,10 @@ limitations under the License.
 */
 
 /*
- * This command filters a JUnit file such that only tests with a name
- * matching a regular expression are passed through. By concatenating
- * multiple input files it is possible to merge them into a single file.
- */
+This command filters a JUnit file such that only tests with a name
+matching a regular expression are passed through. By concatenating
+multiple input files it is possible to merge them into a single file.
+*/
 package main
 
 import (

release-tools/prow.sh

@@ -85,7 +85,7 @@ get_versioned_variable () {
     echo "$value"
 }
 
-configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
 
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because
@@ -218,17 +218,18 @@ configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image ove
 # all generated files are present.
 #
 # CSI_PROW_E2E_REPO=none disables E2E testing.
-# TOOO: remove versioned variables and make e2e version match k8s version
-configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x"
-configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x"
-configvar CSI_PROW_E2E_VERSION_1_17 v1.17.0 "E2E version for Kubernetes 1.17.x"
-# TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases
-configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version
-configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions
-configvar CSI_PROW_E2E_IMPORT_PATH_LATEST k8s.io/kubernetes "E2E package for Kubernetes >= 1.13.x" # currently the same for all versions
-configvar CSI_PROW_E2E_VERSION "$(get_versioned_variable CSI_PROW_E2E_VERSION "${csi_prow_kubernetes_version_suffix}")"  "E2E version"
-configvar CSI_PROW_E2E_REPO "$(get_versioned_variable CSI_PROW_E2E_REPO "${csi_prow_kubernetes_version_suffix}")" "E2E repo"
-configvar CSI_PROW_E2E_IMPORT_PATH "$(get_versioned_variable CSI_PROW_E2E_IMPORT_PATH "${csi_prow_kubernetes_version_suffix}")" "E2E package"
+tag_from_version () {
+    version="$1"
+    shift
+    case "$version" in
+        latest) echo "master";;
+        release-*) echo "$version";;
+        *) echo "v$version";;
+    esac
+}
+configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")"  "E2E version"
+configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo"
+configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 
 # csi-sanity testing from the csi-test repo can be run against the installed
 # CSI driver. For this to work, deploying the driver must expose the Unix domain
@@ -513,6 +514,10 @@ go_version_for_kubernetes () (
     if ! [ "$go_version" ]; then
         die "Unable to determine Go version for Kubernetes $version from hack/lib/golang.sh."
     fi
+    # Strip the trailing .0. Kubernetes includes it, Go itself doesn't.
+    # Ignore: See if you can use ${variable//search/replace} instead.
+    # shellcheck disable=SC2001
+    go_version="$(echo "$go_version" | sed -e 's/\.0$//')"
     echo "$go_version"
 )
 
@@ -1189,3 +1194,23 @@ main () {
 
     return "$ret"
 }
+
+# This function can be called by a repo's top-level cloudbuild.sh:
+# it handles environment set up in the GCR cloud build and then
+# invokes "make push-multiarch" to do the actual image building.
+gcr_cloud_build () {
+    # Register gcloud as a Docker credential helper.
+    # Required for "docker buildx build --push".
+    gcloud auth configure-docker
+
+    if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
+        # Needed for "RUN" steps on non-linux/amd64 platforms.
+        # See https://github.com/multiarch/qemu-user-static#getting-started
+        (set -x; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes)
+    fi
+
+    # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value.
+    REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v')
+
+    run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}"
+}

release-tools/travis.yml

@@ -6,7 +6,7 @@ git:
   depth: false
 matrix:
   include:
-  - go: 1.13.3
+  - go: 1.15
 before_script:
 - mkdir -p bin
 - wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep