diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 57f014ca..91480f30 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -16,6 +16,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run Windows Integration Tests + shell: powershell run: | # required for running Volume and Disk tests Install-WindowsFeature -name Hyper-V-PowerShell diff --git a/release-tools/.github/dependabot.yaml b/release-tools/.github/dependabot.yaml new file mode 100644 index 00000000..814a3449 --- /dev/null +++ b/release-tools/.github/dependabot.yaml @@ -0,0 +1,12 @@ +version: 2 +enable-beta-ecosystems: true +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + labels: + - "area/dependency" + - "release-note-none" + - "ok-to-test" + open-pull-requests-limit: 10 diff --git a/release-tools/.github/workflows/codespell.yml b/release-tools/.github/workflows/codespell.yml new file mode 100644 index 00000000..e74edcef --- /dev/null +++ b/release-tools/.github/workflows/codespell.yml @@ -0,0 +1,15 @@ +# GitHub Action to automate the identification of common misspellings in text files. +# https://github.com/codespell-project/actions-codespell +# https://github.com/codespell-project/codespell +name: codespell +on: [push, pull_request] +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: codespell-project/actions-codespell@master + with: + check_filenames: true + skip: "*.png,*.jpg,*.svg,*.sum,./.git,./.github/workflows/codespell.yml,./prow.sh" diff --git a/release-tools/.github/workflows/trivy.yaml b/release-tools/.github/workflows/trivy.yaml new file mode 100644 index 00000000..47298478 --- /dev/null +++ b/release-tools/.github/workflows/trivy.yaml @@ -0,0 +1,29 @@ +name: Run Trivy scanner for Go version vulnerabilities +on: + push: + branches: + - master + pull_request: +jobs: + trivy: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Go version + id: go-version + run: | + GO_VERSION=$(cat prow.sh | grep "configvar CSI_PROW_GO_VERSION_BUILD" | awk '{print $3}' | sed 's/"//g') + echo "version=$GO_VERSION" >> $GITHUB_OUTPUT + + - name: Run Trivy scanner for Go version vulnerabilities + uses: aquasecurity/trivy-action@master + with: + image-ref: 'golang:${{ steps.go-version.outputs.version }}' + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'library' + severity: 'CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN' diff --git a/release-tools/KUBERNETES_CSI_OWNERS_ALIASES b/release-tools/KUBERNETES_CSI_OWNERS_ALIASES index 344fbe4d..f7bc7018 100644 --- a/release-tools/KUBERNETES_CSI_OWNERS_ALIASES +++ b/release-tools/KUBERNETES_CSI_OWNERS_ALIASES @@ -18,21 +18,23 @@ aliases: # when they are temporarily unable to review PRs. kubernetes-csi-reviewers: - andyzhangx - - chrishenzie + - carlory - ggriffiths - gnufied - humblec + - mauriciopoppe - j-griffith - - Jiawei0227 - jingxu97 - jsafrane - pohly - RaunakShah + - sunnylovestiramisu - xing-yang # This documents who previously contributed to Kubernetes-CSI # as approver. emeritus_approvers: +- Jiawei0227 - lpabon - sbezverk - vladimirvivien diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index 8977fbe6..aab8d6e2 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -17,7 +17,7 @@ The release manager must: Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs must be updated. -[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the +[Our CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) have the naming convention `-on-`. 1. Jobs should be actively monitored to find and fix failures in sidecars and @@ -46,52 +46,51 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download the latest version of the - [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) -1. Create a - [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) - with `repo:public_repo` access -1. Generate release notes for the release. Replace arguments with the relevant - information. - * Clean up old cached information (also needed if you are generating release - notes for multiple repos) - ```bash - rm -rf /tmp/k8s-repo - ``` - * For new minor releases on master: - ```bash - GITHUB_TOKEN= release-notes \ - --discover=mergebase-to-latest \ - --org=kubernetes-csi \ - --repo=external-provisioner \ - --required-author="" \ - --markdown-links \ - --output out.md - ``` - * For new patch releases on a release branch: - ```bash - GITHUB_TOKEN= release-notes \ - --discover=patch-to-latest \ - --branch=release-1.1 \ - --org=kubernetes-csi \ - --repo=external-provisioner \ - --required-author="" \ - --markdown-links \ - --output out.md - ``` -1. Compare the generated output to the new commits for the release to check if - any notable change missed a release note. -1. Reword release notes as needed. Make sure to check notes for breaking - changes and deprecations. -1. If release is a new major/minor version, create a new `CHANGELOG-..md` - file. Otherwise, add the release notes to the top of the existing CHANGELOG - file for that minor version. -1. Submit a PR for the CHANGELOG changes. -1. Submit a PR for README changes, in particular, Compatibility, Feature status, - and any other sections that may need updating. +1. Update dependencies for sidecars + 1. For new minor versions, use + [go-modules-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-update.sh), + 1. For CVE fixes on patch versions, use + [go-modules-targeted-update.sh](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/go-modules-targeted-update.sh), + Read the instructions at the top of the script. 1. Check that all [canary CI - jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, + jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing, and that test coverage is adequate for the changes that are going into the release. +1. Check that the post-\-push-images builds are succeeding. + [Example](https://testgrid.k8s.io/sig-storage-image-build#post-external-snapshotter-push-images) +1. Generate release notes. + 1. Download the latest version of the [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes) + 1. Create a + [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) + with `repo:public_repo` access + 1. For patch release, use the script generate_patch_release_notes.sh. Read the instructions at the top of the + script. The script also creates PRs for each branch. + 1. For new minor releases, follow these steps and replace arguments with the relevant + information. + * Clean up old cached information (also needed if you are generating release + notes for multiple repos) + ```bash + rm -rf /tmp/k8s-repo + ``` + * For new minor releases on master: + ```bash + GITHUB_TOKEN= release-notes \ + --discover=mergebase-to-latest \ + --org=kubernetes-csi \ + --repo=external-provisioner \ + --required-author="" \ + --markdown-links \ + --output out.md + ``` + 1. Compare the generated output to the new commits for the release to check if + any notable change missed a release note. + 1. Reword release notes as needed, ideally in the original PRs so that the + release notes can be regenerated. Make sure to check notes for breaking + changes and deprecations. + 1. If release is a new major/minor version, create a new `CHANGELOG-..md` + file. + 1. Submit a PR for the CHANGELOG changes. +1. Submit a PR for README changes, in particular, Compatibility, Feature status, + and any other sections that may need updating. 1. Make sure that no new PRs have merged in the meantime, and no PRs are in flight and soon to be merged. 1. Create a new release following a previous release as a template. Be sure to select the correct @@ -99,10 +98,10 @@ naming convention `-on-`. [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new) 1. If release was a new major/minor version, create a new `release-` branch at that commit. -1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build). -1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From +1. Check [image build status](https://testgrid.k8s.io/sig-storage-image-build). +1. Promote images from k8s-staging-sig-storage to registry.k8s.io/sig-storage. From the [k8s image - repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage), + repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage), run `./generate.sh > images.yaml`, and send a PR with the updated images. Once merged, the image promoter will copy the images from staging to prod. 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar @@ -118,7 +117,7 @@ naming convention `-on-`. The following jobs are triggered after tagging to produce the corresponding image(s): -https://k8s-testgrid.appspot.com/sig-storage-image-build +https://testgrid.k8s.io/sig-storage-image-build Clicking on a failed build job opens that job in https://prow.k8s.io. Next to the job title is a rerun icon (circle with arrow). Clicking it opens a popup diff --git a/release-tools/build.make b/release-tools/build.make index ff428890..39a34777 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -45,9 +45,10 @@ REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git r # Determined dynamically. IMAGE_TAGS= -# A "canary" image gets built if the current commit is the head of the remote "master" branch. +# A "canary" image gets built if the current commit is the head of the remote "master" or "main" branch. # That branch does not exist when building some other branch in TravisCI. IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/master 2>/dev/null)" ]; then echo "canary"; fi) +IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/main 2>/dev/null)" ]; then echo "canary"; fi) # A "X.Y.Z-canary" image gets built if the current commit is the head of a "origin/release-X.Y.Z" branch. # The actual suffix does not matter, only the "release-" prefix is checked. @@ -62,9 +63,9 @@ IMAGE_NAME=$(REGISTRY_NAME)/$* ifdef V # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed. -TESTARGS = -v -args -alsologtostderr -v 5 +TESTARGS = -race -v -args -alsologtostderr -v 5 else -TESTARGS = +TESTARGS = -race endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables @@ -143,12 +144,12 @@ DOCKER_BUILDX_CREATE_ARGS ?= # Windows binaries can be built before adding a Dockerfile for it. # # BUILD_PLATFORMS determines which individual images are included in the multiarch image. -# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines +# PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name, and determines # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ export DOCKER_CLI_EXPERIMENTAL=enabled; \ - docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest --driver-opt image=moby/buildkit:v0.10.6; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ @@ -191,7 +192,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ - if [ $(PULL_BASE_REF) = "master" ]; then \ + if [ $(PULL_BASE_REF) = "master" ] || [ $(PULL_BASE_REF) = "main" ]; then \ : "creating or overwriting canary image"; \ pushMultiArch canary; \ elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ @@ -209,7 +210,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% .PHONY: check-pull-base-ref check-pull-base-ref: if ! [ "$(PULL_BASE_REF)" ]; then \ - echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \ + echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name."; \ exit 1; \ fi @@ -322,3 +323,10 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" + +# Test klog usage. This test is optional and must be explicitly added to `test` target in the main Makefile: +# test: test-logcheck +.PHONY: test-logcheck +test-logcheck: + @ echo; echo "### $@:" + @ ./release-tools/verify-logcheck.sh diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml index 4bcffad3..1693df84 100644 --- a/release-tools/cloudbuild.yaml +++ b/release-tools/cloudbuild.yaml @@ -13,7 +13,7 @@ # See https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. # -# To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage. +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). # Building three images in external-snapshotter takes more than an hour. @@ -26,7 +26,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20210917-12df099d55' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20240718-5ef92b5c36' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/release-tools/contrib/get_supported_version_csi-sidecar.py b/release-tools/contrib/get_supported_version_csi-sidecar.py new file mode 100644 index 00000000..30bbcf24 --- /dev/null +++ b/release-tools/contrib/get_supported_version_csi-sidecar.py @@ -0,0 +1,170 @@ +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import datetime +import re +from collections import defaultdict +import subprocess +import shutil +from dateutil.relativedelta import relativedelta + +def check_gh_command(): + """ + Pretty much everything is processed from `gh` + Check that the `gh` command is in the path before anything else + """ + if not shutil.which('gh'): + print("Error: The `gh` command is not available in the PATH.") + print("Please install the GitHub CLI (https://cli.github.com/) and try again.") + exit(1) + +def duration_ago(dt): + """ + Humanize duration outputs + """ + delta = relativedelta(datetime.datetime.now(), dt) + if delta.years > 0: + return f"{delta.years} year{'s' if delta.years > 1 else ''} ago" + elif delta.months > 0: + return f"{delta.months} month{'s' if delta.months > 1 else ''} ago" + elif delta.days > 0: + return f"{delta.days} day{'s' if delta.days > 1 else ''} ago" + elif delta.hours > 0: + return f"{delta.hours} hour{'s' if delta.hours > 1 else ''} ago" + elif delta.minutes > 0: + return f"{delta.minutes} minute{'s' if delta.minutes > 1 else ''} ago" + else: + return "just now" + +def parse_version(version): + """ + Parse version assuming it is in the form of v1.2.3 + """ + pattern = r"v(\d+)\.(\d+)\.(\d+)" + match = re.match(pattern, version) + if match: + major, minor, patch = map(int, match.groups()) + return (major, minor, patch) + +def end_of_life_grouped_versions(versions): + """ + Calculate the end of life date for a minor release version according to : https://kubernetes-csi.github.io/docs/project-policies.html#support + + The input is an array of tuples of: + * grouped versions (e.g. 1.0, 1.1) + * array of that contains all versions and their release date (e.g. 1.0.0, 01-01-2013) + + versions structure example : + [((3, 5), [('v3.5.0', datetime.datetime(2023, 4, 27, 22, 28, 6))]), + ((3, 4), + [('v3.4.1', datetime.datetime(2023, 4, 5, 17, 41, 15)), + ('v3.4.0', datetime.datetime(2022, 12, 27, 23, 43, 41))])] + """ + supported_versions = [] + # Prepare dates for later calculation + now = datetime.datetime.now() + one_year = datetime.timedelta(days=365) + three_months = datetime.timedelta(days=90) + + # get the newer versions on top + sorted_versions_list = sorted(versions.items(), key=lambda x: x[0], reverse=True) + + # the latest version is always supported no matter the release date + latest = sorted_versions_list.pop(0) + supported_versions.append(latest[1][-1]) + + for v in sorted_versions_list: + first_release = v[1][-1] + last_release = v[1][0] + # if the release is less than a year old we support the latest patch version + if now - first_release[1] < one_year: + supported_versions.append(last_release) + # if the main release is older than a year and has a recent path, this is supported + elif now - last_release[1] < three_months: + supported_versions.append(last_release) + return supported_versions + +def get_release_docker_image(repo, version): + """ + Extract docker image name from the release page documentation + """ + output = subprocess.check_output(['gh', 'release', '-R', repo, 'view', version], text=True) + #Extract matching image name excluding ` + match = re.search(r"docker pull ([\.\/\-\:\w\d]*)", output) + docker_image = match.group(1) if match else '' + return((version, docker_image)) + +def get_versions_from_releases(repo): + """ + Using `gh` cli get the github releases page details then + create a list of grouped version on major.minor + and for each give all major.minor.patch with release dates + """ + # Run the `gh release` command to get the release list + output = subprocess.check_output(['gh', 'release', '-R', repo, 'list'], text=True) + # Parse the output and group by major and minor version numbers + versions = defaultdict(lambda: []) + for line in output.strip().split('\n'): + parts = line.split('\t') + # pprint.pprint(parts) + version = parts[0] + parsed_version = parse_version(version) + if parsed_version is None: + continue + major, minor, patch = parsed_version + + published = datetime.datetime.strptime(parts[3], '%Y-%m-%dT%H:%M:%SZ') + versions[(major, minor)].append((version, published)) + return(versions) + + +def main(): + manual = """ + This script lists the supported versions Github releases according to https://kubernetes-csi.github.io/docs/project-policies.html#support + It has been designed to help to update the tables from : https://kubernetes-csi.github.io/docs/sidecar-containers.html\n\n + It can take multiple repos as argument, for all CSI sidecars details you can run: + ./get_supported_version_csi-sidecar.py -R kubernetes-csi/external-attacher -R kubernetes-csi/external-provisioner -R kubernetes-csi/external-resizer -R kubernetes-csi/external-snapshotter -R kubernetes-csi/livenessprobe -R kubernetes-csi/node-driver-registrar -R kubernetes-csi/external-health-monitor\n + With the output you can then update the documentation manually. + """ + parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=manual) + parser.add_argument('--repo', '-R', required=True, action='append', dest='repos', help='The name of the repository in the format owner/repo.') + parser.add_argument('--display', '-d', action='store_true', help='(default) Display EOL versions with their dates', default=True) + parser.add_argument('--doc', '-D', action='store_true', help='Helper to https://kubernetes-csi.github.io/docs/ that prints Docker image for each EOL version') + + args = parser.parse_args() + + # Verify pre-reqs + check_gh_command() + + # Process all repos + for repo in args.repos: + versions = get_versions_from_releases(repo) + eol_versions = end_of_life_grouped_versions(versions) + + if args.display: + print(f"Supported versions with release date and age of `{repo}`:\n") + for version in eol_versions: + print(f"{version[0]}\t{version[1].strftime('%Y-%m-%d')}\t{duration_ago(version[1])}") + + # TODO : generate proper doc output for the tables of: https://kubernetes-csi.github.io/docs/sidecar-containers.html + if args.doc: + print("\nSupported Versions with docker images for each end of life version:\n") + for version in eol_versions: + _, image = get_release_docker_image(repo, version[0]) + print(f"{version[0]}\t{image}") + print() + +if __name__ == '__main__': + main() diff --git a/release-tools/filter-junit.go b/release-tools/filter-junit.go index 5454092b..aab1b8b6 100644 --- a/release-tools/filter-junit.go +++ b/release-tools/filter-junit.go @@ -24,7 +24,6 @@ package main import ( "encoding/xml" "flag" - "io/ioutil" "os" "regexp" ) @@ -56,6 +55,7 @@ type TestCase struct { Name string `xml:"name,attr"` Time string `xml:"time,attr"` SystemOut string `xml:"system-out,omitempty"` + SystemErr string `xml:"system-err,omitempty"` Failure string `xml:"failure,omitempty"` Skipped SkipReason `xml:"skipped,omitempty"` } @@ -95,7 +95,7 @@ func main() { } } else { var err error - data, err = ioutil.ReadFile(input) + data, err = os.ReadFile(input) if err != nil { panic(err) } @@ -109,7 +109,7 @@ func main() { if err := xml.Unmarshal(data, &junitv2); err != nil { panic(err) } - junit = junitv2.TestSuite + junit.TestCases = append(junit.TestCases, junitv2.TestSuite.TestCases...) } } @@ -142,7 +142,7 @@ func main() { panic(err) } } else { - if err := ioutil.WriteFile(*output, data, 0644); err != nil { + if err := os.WriteFile(*output, data, 0644); err != nil { panic(err) } } diff --git a/release-tools/generate-patch-release-notes.sh b/release-tools/generate-patch-release-notes.sh new file mode 100755 index 00000000..536a1490 --- /dev/null +++ b/release-tools/generate-patch-release-notes.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Usage: generate_patch_release_notes.sh +# +# Generates and creates PRs for kubernetes-csi patch releases. +# +# Required environment variables +# CSI_RELEASE_TOKEN: Github token needed for generating release notes +# GITHUB_USER: Github username to create PRs with +# +# Required tools: +# - gh +# - release-notes (https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md) +# +# Instructions: +# 1. Install the required tools +# 2. Login with "gh auth login" +# 3. Copy this script to the kubernetes-csi directory (one directory above the repos) +# 4. Update the repos and versions in the $releases array +# 5. Set environment variables +# 6. Run script from the kubernetes-csi directory +# +# Caveats: +# - This script doesn't handle regenerating and updating existing PRs yet. +# It might work if you comment out the PR creation line + +set -e +set -x + +releases=( +# "external-attacher 4.4.1" +# "external-provisioner 3.6.1" +# "external-snapshotter 6.2.3" +) + +function gen_patch_relnotes() { + rm out.md || true + rm -rf /tmp/k8s-repo || true + GITHUB_TOKEN="$CSI_RELEASE_TOKEN" \ + release-notes --start-rev="$3" --end-rev="$2" --branch="$2" \ + --org=kubernetes-csi --repo="$1" \ + --required-author="" --markdown-links --output out.md +} + +for rel in "${releases[@]}"; do + read -r repo version <<< "$rel" + + # Parse minor version + minorPatchPattern="(^[[:digit:]]+\.[[:digit:]]+)\.([[:digit:]]+)" + [[ "$version" =~ $minorPatchPattern ]] + minor="${BASH_REMATCH[1]}" + patch="${BASH_REMATCH[2]}" + + echo "$repo $version $minor $patch" + prevPatch="$((patch-1))" + prevVer="v$minor.$prevPatch" + + pushd "$repo/CHANGELOG" + + git fetch upstream + + # Create branch + branch="changelog-release-$minor" + git checkout master + git branch -D "$branch" || true + git checkout --track "upstream/release-$minor" -b "$branch" + + # Generate release notes + gen_patch_relnotes "$repo" "release-$minor" "$prevVer" + cat > tmp.md <> tmp.md + echo >> tmp.md + rm out.md + + file="CHANGELOG-$minor.md" + cat "$file" >> tmp.md + mv tmp.md "$file" + + git add -u + git commit -m "Add changelog for $version" + git push -f origin "$branch" + + # Create PR +prbody=$(cat </dev/null)" ]; then + git checkout master && git branch -D "module-update-$branch" + fi + git checkout -B "module-update-$branch" "upstream/$branch" + + for mod in "${modules[@]}"; do + go get "$mod" + done + go mod tidy + go mod vendor + + git add --all + git commit -m "Update go modules" + git push origin "module-update-$branch" --force + + # Create PR +prbody=$(cat </dev/null)" ]; then + git checkout master && git branch -d "module-update-$i" + fi + git checkout -B "module-update-$i" "origin/$i" + rm -rf .git/MERGE* + if ! git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master; then + # Sometimes "--squash" leads to merge conflicts. Because we know that "release-tools" + # is an unmodified copy of csi-release-tools, we can automatically resolve that + # by replacing it completely. + if [ -e .git/MERGE_MSG ] && [ -e .git/FETCH_HEAD ] && grep -q "^# Conflict" .git/MERGE_MSG; then + rm -rf release-tools + mkdir release-tools + git archive FETCH_HEAD | tar -C release-tools -xf - + git add release-tools + git commit --file=.git/MERGE_MSG + else + exit 1 + fi + fi + RETRY=0 + while ! ./release-tools/go-get-kubernetes.sh -p "$v" && RETRY < $MAX_RETRY + do + RETRY=$((RETRY+1)) + go mod tidy && go mod vendor && go mod tidy + done + go mod tidy && go mod vendor && go mod tidy + git add --all + git commit -m "Update dependency go modules for k8s v$v" + git remote set-url origin "https://github.com/$username/$repo.git" + make test + git push origin "module-update-$i" --force + # Create PR +prbody=$(cat < as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.8.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.17.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" @@ -228,8 +231,11 @@ configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION} configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" -# Local path for e2e tests. Set to "none" to disable. -configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" +# Local path & package path for e2e tests. Set to "none" to disable. +# When using versioned go modules, the import path is the module path whereas the path +# should not contain the version and be the directory where the module is checked out. +configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package (go import path)" +configvar CSI_PROW_SIDECAR_E2E_PATH "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" "CSI Sidecar E2E path (directory)" # csi-sanity testing from the csi-test repo can be run against the installed # CSI driver. For this to work, deploying the driver must expose the Unix domain @@ -237,7 +243,7 @@ configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v5.0.0 "csi-test version" +configvar CSI_PROW_SANITY_VERSION v5.3.1 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -245,7 +251,7 @@ configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI dr # The version of dep to use for 'make test-vendor'. Ignored if the project doesn't # use dep. Only binary releases of dep are supported (https://github.com/golang/dep/releases). -configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor checking" +configvar CSI_PROW_DEP_VERSION v0.5.4 "golang dep version to be used for vendor checking" # Each job can run one or more of the following tests, identified by # a single word: @@ -419,7 +425,7 @@ die () { exit 1 } -# Ensure that PATH has the desired version of the Go tools, then run command given as argument. +# Ensure we use the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. run_with_go () { @@ -427,15 +433,16 @@ run_with_go () { version="$1" shift - if ! [ "$version" ] || go version 2>/dev/null | grep -q "go$version"; then - run "$@" - else - if ! [ -d "${CSI_PROW_WORK}/go-$version" ]; then - run curl --fail --location "https://dl.google.com/go/go$version.linux-amd64.tar.gz" | tar -C "${CSI_PROW_WORK}" -zxf - || die "installation of Go $version failed" - mv "${CSI_PROW_WORK}/go" "${CSI_PROW_WORK}/go-$version" + if [ "$version" ]; then + version=go$version + if [ "$(GOTOOLCHAIN=$version go version | cut -d' ' -f3)" != "$version" ]; then + die "Please install Go 1.21+" fi - PATH="${CSI_PROW_WORK}/go-$version/bin:$PATH" run "$@" + else + version=local fi + # Set GOMODCACHE to make sure Kubernetes does not need to download again. + GOTOOLCHAIN=$version GOMODCACHE="$(go env GOMODCACHE)" run "$@" } # Ensure that we have the desired version of kind. @@ -469,7 +476,7 @@ install_dep () { if dep version 2>/dev/null | grep -q "version:.*${CSI_PROW_DEP_VERSION}$"; then return fi - run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/v0.5.4/dep-linux-amd64" && + run curl --fail --location -o "${CSI_PROW_WORK}/bin/dep" "https://github.com/golang/dep/releases/download/${CSI_PROW_DEP_VERSION}/dep-linux-amd64" && chmod u+x "${CSI_PROW_WORK}/bin/dep" } @@ -561,7 +568,15 @@ go_version_for_kubernetes () ( local version="$2" local go_version - # We use the minimal Go version specified for each K8S release (= minimum_go_version in hack/lib/golang.sh). + # Try to get the version for .go-version + go_version="$( cat "$path/.go-version" )" + if [ "$go_version" ]; then + echo "$go_version" + return + fi + + # Fall back to hack/lib/golang.sh parsing. + # This is necessary in v1.26.0 and older Kubernetes releases that do not have .go-version. # More recent versions might also work, but we don't want to count on that. go_version="$(grep minimum_go_version= "$path/hack/lib/golang.sh" | sed -e 's/.*=go//')" if ! [ "$go_version" ]; then @@ -610,7 +625,7 @@ start_cluster () { go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 # shellcheck disable=SC2046 - (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image "${CSI_PROW_WORK}/src/kubernetes" --image csiprow/node:latest) || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" @@ -872,10 +887,17 @@ install_snapshot_controller() { cnt=0 expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML") expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML") - while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do + expect_key='app\.kubernetes\.io/name' + expected_label=$(kubectl apply --dry-run=client -o "jsonpath={.spec.template.metadata.labels['$expect_key']}" -f "$SNAPSHOT_CONTROLLER_YAML") + if [ -z "${expected_label}" ]; then + expect_key='app' + expected_label=$(kubectl apply --dry-run=client -o "jsonpath={.spec.template.metadata.labels['$expect_key']}" -f "$SNAPSHOT_CONTROLLER_YAML") + fi + expect_key=${expect_key//\\/} + while [ "$(kubectl get pods -n "$expected_namespace" -l "$expect_key"="$expected_label" | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then echo "snapshot-controller pod status:" - kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller + kubectl describe pods -n "$expected_namespace" -l "$expect_key"="$expected_label" echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi @@ -1008,17 +1030,20 @@ run_e2e () ( # the full Kubernetes E2E testsuite while only running a few tests. move_junit () { if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then - run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml + mkdir -p "${ARTIFACTS}/junit/${name}" && + mkdir -p "${ARTIFACTS}/junit/steps" && + run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && + mv "${ARTIFACTS}"/junit_[0-9]*.xml "${ARTIFACTS}/junit/${name}/" fi } trap move_junit EXIT if [ "${name}" == "local" ]; then - cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" && - run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local + cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_PATH}" && + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local else cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" fi ) @@ -1085,13 +1110,14 @@ kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PA EOF chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh + mkdir -p "${ARTIFACTS}/junit/steps" # This cannot run in parallel, because -csi.junitfile output # from different Ginkgo nodes would go to the same file. Also the # staging and target directories are the same. run_with_loggers "${CSI_PROW_WORK}/csi-sanity" \ -ginkgo.v \ - -csi.junitfile "${ARTIFACTS}/junit_sanity.xml" \ + -csi.junitfile "${ARTIFACTS}/junit/steps/junit_sanity.xml" \ -csi.endpoint "dns:///$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' csi-prow-control-plane):$(kubectl get "services/${CSI_PROW_SANITY_SERVICE}" -o "jsonpath={..nodePort}")" \ -csi.stagingdir "/tmp/staging" \ -csi.mountdir "/tmp/mount" \ @@ -1121,7 +1147,8 @@ make_test_to_junit () { # Plain make-test.xml was not delivered as text/xml by the web # server and ignored by spyglass. It seems that the name has to # match junit*.xml. - out="${ARTIFACTS}/junit_make_test.xml" + out="${ARTIFACTS}/junit/steps/junit_make_test.xml" + mkdir -p "$(dirname "$out")" testname= echo "" >>"$out" @@ -1305,7 +1332,7 @@ main () { if tests_enabled "parallel"; then # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel ${CSI_PROW_GINKGO_PARALLEL} \ -focus="$focus" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel failed" @@ -1315,7 +1342,7 @@ main () { # Run tests that are feature tagged, but non-alpha # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-features ${CSI_PROW_GINKGO_PARALLEL} \ -focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" @@ -1363,7 +1390,7 @@ main () { if tests_enabled "parallel-alpha"; then # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-alpha ${CSI_PROW_GINKGO_PARALLEL} \ -focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then warn "E2E parallel alpha failed" @@ -1385,8 +1412,8 @@ main () { fi # Merge all junit files into one. This gets rid of duplicated "skipped" tests. - if ls "${ARTIFACTS}"/junit_*.xml 2>/dev/null >&2; then - run_filter_junit -o "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}"/junit_*.xml && rm "${ARTIFACTS}"/junit_*.xml && mv "${CSI_PROW_WORK}/junit_final.xml" "${ARTIFACTS}" + if ls "${ARTIFACTS}"/junit/steps/junit_*.xml 2>/dev/null >&2; then + run_filter_junit -o "${ARTIFACTS}/junit_final.xml" "${ARTIFACTS}"/junit/steps/junit_*.xml fi return "$ret" diff --git a/release-tools/pull-test.sh b/release-tools/pull-test.sh index b019c177..80317720 100755 --- a/release-tools/pull-test.sh +++ b/release-tools/pull-test.sh @@ -20,11 +20,17 @@ set -ex +# Prow checks out repos with --filter=blob:none. This breaks +# "git subtree pull" unless we enable fetching missing file content. +GIT_NO_LAZY_FETCH=0 +export GIT_NO_LAZY_FETCH + # It must be called inside the updated csi-release-tools repo. CSI_RELEASE_TOOLS_DIR="$(pwd)" # Update the other repo. cd "$PULL_TEST_REPO_DIR" +git reset --hard # Shouldn't be necessary, but somehow is to avoid "fatal: working tree has modifications. Cannot add." (https://stackoverflow.com/questions/3623351/git-subtree-pull-says-that-the-working-tree-has-modifications-but-git-status-sa) git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master git log -n2 diff --git a/release-tools/verify-logcheck.sh b/release-tools/verify-logcheck.sh new file mode 100755 index 00000000..f3287e7b --- /dev/null +++ b/release-tools/verify-logcheck.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script uses the logcheck tool to analyze the source code +# for proper usage of klog contextual logging. + +set -o errexit +set -o nounset +set -o pipefail + +LOGCHECK_VERSION=${1:-0.8.2} + +# This will canonicalize the path +CSI_LIB_UTIL_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P) + +# Create a temporary directory for installing logcheck and +# set up a trap command to remove it when the script exits. +CSI_LIB_UTIL_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t csi-lib-utils.XXXXXX) +trap 'rm -rf "${CSI_LIB_UTIL_TEMP}"' EXIT + +echo "Installing logcheck to temp dir: sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" +GOBIN="${CSI_LIB_UTIL_TEMP}" go install "sigs.k8s.io/logtools/logcheck@v${LOGCHECK_VERSION}" +echo "Verifying logcheck: ${CSI_LIB_UTIL_TEMP}/logcheck -check-contextual ${CSI_LIB_UTIL_ROOT}/..." +"${CSI_LIB_UTIL_TEMP}/logcheck" -check-contextual -check-with-helpers "${CSI_LIB_UTIL_ROOT}/..."