diff --git a/Makefile b/Makefile index 6add1025f70..4d96832cc1b 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,10 @@ ifeq ($(CI),true) $(shell git config --global --add safe.directory '*') endif -GO_INSTALL = ./hack/go-install.sh - -TOOLS_DIR=hack/tools +TOOLS_DIR = hack/tools +export UGET_DIRECTORY = $(TOOLS_DIR) +export UGET_CHECKSUMS = hack/tools.checksums +export UGET_VERSIONED_BINARIES = true ROOT_DIR=$(abspath .) TOOLS_GOBIN_DIR := $(abspath $(TOOLS_DIR)) GOBIN_DIR=$(abspath ./bin) @@ -50,20 +51,20 @@ YAML_PATCH_BIN := yaml-patch YAML_PATCH := $(TOOLS_DIR)/$(YAML_PATCH_BIN)-$(YAML_PATCH_VER) export YAML_PATCH # so hack scripts can use it -GOLANGCI_LINT_VER := v2.1.6 +GOLANGCI_LINT_VER := 2.1.6 GOLANGCI_LINT_BIN := golangci-lint GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) GOLANGCI_LINT_FLAGS ?= -HTTEST_VER := v0.3.2 +HTTEST_VER := 0.3.4 HTTEST_BIN := httest -HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) +export HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) -GOTESTSUM_VER := v1.12.3 +GOTESTSUM_VER := 1.12.3 GOTESTSUM_BIN := gotestsum GOTESTSUM := $(abspath $(TOOLS_DIR))/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER) -LOGCHECK_VER := v0.9.0 +LOGCHECK_VER := d35c84c015fe03a1421e5f2ce1e3c0c3bc38d077 LOGCHECK_BIN := logcheck LOGCHECK := $(TOOLS_GOBIN_DIR)/$(LOGCHECK_BIN)-$(LOGCHECK_VER) export LOGCHECK # so hack scripts can use it @@ -129,14 +130,29 @@ install: require-jq require-go require-git verify-go-versions ## Install the pro .PHONY: install $(GOLANGCI_LINT): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/v2/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + @hack/uget.sh \ + https://github.com/golangci/golangci-lint/releases/download/v{VERSION}/golangci-lint-{VERSION}-{GOOS}-{GOARCH}.tar.gz \ + ${GOLANGCI_LINT_BIN} \ + ${GOLANGCI_LINT_VER} + +# help staging modules to install the same linter version +golangci-lint-version: + @echo $(GOLANGCI_LINT_VER) +.PHONY: golangci-lint-version $(HTTEST): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) go.xrstf.de/httest $(HTTEST_BIN) $(HTTEST_VER) + @hack/uget.sh \ + https://codeberg.org/xrstf/httest/releases/download/v{VERSION}/httest_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ + ${HTTEST_BIN} \ + ${HTTEST_VER} $(LOGCHECK): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) sigs.k8s.io/logtools/logcheck $(LOGCHECK_BIN) $(LOGCHECK_VER) + @GO_MODULE=true hack/uget.sh \ + sigs.k8s.io/logtools/logcheck \ + ${LOGCHECK_BIN} \ + $(LOGCHECK_VER) +.PHONY: $(KCP_APIGEN_GEN) $(KCP_APIGEN_GEN): pushd . && cd staging/src/github.com/kcp-dev/sdk && GOBIN=$(TOOLS_GOBIN_DIR) go install ./cmd/apigen && popd @@ -197,13 +213,17 @@ tools: $(GOLANGCI_LINT) $(HTTEST) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PAT .PHONY: tools $(CONTROLLER_GEN): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) sigs.k8s.io/controller-tools/cmd/controller-gen $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER) + @UNCOMPRESSED=true hack/uget.sh https://github.com/kubernetes-sigs/controller-tools/releases/download/{VERSION}/controller-gen-{GOOS}-{GOARCH} ${CONTROLLER_GEN_BIN} $(CONTROLLER_GEN_VER) controller-gen* $(YAML_PATCH): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) + @GO_MODULE=true hack/uget.sh github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) $(GOTESTSUM): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) gotest.tools/gotestsum $(GOTESTSUM_BIN) $(GOTESTSUM_VER) + @hack/uget.sh \ + https://github.com/gotestyourself/gotestsum/releases/download/v{VERSION}/gotestsum_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ + ${GOTESTSUM_BIN} \ + ${GOTESTSUM_VER} \ + ${GOTESTSUM_BIN} crds: $(CONTROLLER_GEN) $(YAML_PATCH) ## Generate crds ./hack/update-codegen-crds.sh @@ -267,7 +287,7 @@ BOILERPLATE_KUBERNETES_FILES := $(abspath ./hack/boilerplate/boilerplate_kuberne .PHONY: verify-boilerplate verify-boilerplate: ## Verify boilerplate - hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate --skip docs/venv --skip pkg/network/dialer --skip-files-list $(BOILERPLATE_MODIFIED_FILES) --skip-files-list $(BOILERPLATE_KUBERNETES_FILES) + hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate --skip hack/uget.sh --skip docs/venv --skip pkg/network/dialer --skip-files-list $(BOILERPLATE_MODIFIED_FILES) --skip-files-list $(BOILERPLATE_KUBERNETES_FILES) hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate/boilerplate_modified --filenames-list $(BOILERPLATE_MODIFIED_FILES) hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate/boilerplate_kubernetes --filenames-list $(BOILERPLATE_KUBERNETES_FILES) diff --git a/hack/go-install.sh b/hack/go-install.sh deleted file mode 100755 index ef6f8af6345..00000000000 --- a/hack/go-install.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The KCP Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Originally copied from -# https://github.com/kubernetes-sigs/cluster-api-provider-gcp/blob/c26a68b23e9317323d5d37660fe9d29b3d2ff40c/scripts/go_install.sh - -set -o errexit -set -o nounset -set -o pipefail - -if [[ -z "${1:-}" ]]; then - echo "must provide module as first parameter" - exit 1 -fi - -if [[ -z "${2:-}" ]]; then - echo "must provide binary name as second parameter" - exit 1 -fi - -if [[ -z "${3:-}" ]]; then - echo "must provide version as third parameter" - exit 1 -fi - -if [[ -z "${GOBIN:-}" ]]; then - echo "GOBIN is not set. Must set GOBIN to install the bin in a specified directory." - exit 1 -fi - -mkdir -p "${GOBIN}" - -tmp_dir=$(mktemp -d -t goinstall_XXXXXXXXXX) -function clean { - rm -rf "${tmp_dir}" -} -trap clean EXIT - -rm "${GOBIN}/${2}"* > /dev/null 2>&1 || true - -cd "${tmp_dir}" - -# create a new module in the tmp directory -go mod init fake/mod - -# install the golang module specified as the first argument -go install -tags kcptools "${1}@${3}" -mv "${GOBIN}/${2}" "${GOBIN}/${2}-${3}" -ln -sf "${GOBIN}/${2}-${3}" "${GOBIN}/${2}" \ No newline at end of file diff --git a/hack/tools.checksums b/hack/tools.checksums new file mode 100644 index 00000000000..bc447c4f91f --- /dev/null +++ b/hack/tools.checksums @@ -0,0 +1,15 @@ +controller-gen|GOARCH=amd64;GOOS=linux|21e5f3239666fc0c5e2d23c2a3a83fd655af40a969ede7a118b86832c35a829f +controller-gen|GOARCH=arm64;GOOS=darwin|2ca28be7185d9279ed82e3355529b0543938f392cb812add3f25a62196ed7441 +controller-gen|GOARCH=arm64;GOOS=linux|a1a1f758435d05933c4b2f8c292f8ab2448e81a02c45f14dbd81c10e87ec4b20 +golangci-lint|GOARCH=amd64;GOOS=linux|7009324a8aad93c1f84dce1c3cf61181bdd6b68e5f1b8b5d6971662258255050 +golangci-lint|GOARCH=arm64;GOOS=darwin|96dc1dbbb686fc7982b9ea4b800f3d234a2d9fb78b0344f287e8d28ee11bc72b +golangci-lint|GOARCH=arm64;GOOS=linux|c51ff5b21be688b043baea44de7dd855cf07b855c14f0de405bfaf922b1d7634 +gotestsum|GOARCH=amd64;GOOS=linux|2e505a9368568aa7422e0a90ef77acc8807c0d3272ab81c7a69e3e8688d1cf65 +gotestsum|GOARCH=arm64;GOOS=darwin|020be8d14358c7ac4155e296436057cf4b1f1232f8f8f3d71f22a0e7a5504340 +gotestsum|GOARCH=arm64;GOOS=linux|2f8517768c2831750cb372e379404a059dbd20f2b1f79bcc235c4cab4540cb10 +httest|GOARCH=amd64;GOOS=linux|d524eea4fa0bcb9376466a6da7892d88556ed6c03d70f282946e5ca8474b4232 +httest|GOARCH=arm64;GOOS=darwin|12ac43f851459bc1e82302ff2d2bde987970c2c0a47933f79ddd3c277bf4a93d +httest|GOARCH=arm64;GOOS=linux|6f317e0e9857a7582bbf881d30c8c64ead353312d3141b31cf1261de002d91ea +yaml-patch|GOARCH=amd64;GOOS=linux|6eae938136a18a5730700676f89ce7a0d4a89d332219fad180202b329501a88b +yaml-patch|GOARCH=arm64;GOOS=darwin|ce2f9448472b0e69e44c28c7e6879caae46664aa61e251fe7f181b78158ff9b0 +yaml-patch|GOARCH=arm64;GOOS=linux|f379170ba41d9d02446cbdf42fd48b47892a23f9333d9385ad8306449fc22d8b diff --git a/hack/uget.sh b/hack/uget.sh new file mode 100755 index 00000000000..c59c0a4000c --- /dev/null +++ b/hack/uget.sh @@ -0,0 +1,611 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget +# SPDX-License-Identifier: MIT +# +# µget 0.3.3 – your friendly downloader +# ------------------------------------- +# +# µget can download software as binaries, archives or Go modules. +# +# Usage: ./uget.sh URL_PATTERN BINARY_NAME VERSION [EXTRACT_PATTERN=**/$BINARY_NAME] + +set -eu + +############################################################################### +# Configuration + +# µget supports a large range of environment variables to customize its +# behaviour. Global configuration settings are prefixed with UGET_, settings +# usually meant for a single tool only have no prefix (like GO_MODULE). + +# GO_MODULE can be set to true to create a dummy Go module, add the given +# URL and version as a dependency and then go build the desired binary. Note +# that Go modules do not use checksums by default, see $UGET_GO_CHECKSUMS. +GO_MODULE=${GO_MODULE:-false} + +# UNCOMPRESSED can be set to true if the downloaded file is not an archive, +# but the binary itself and doesn't need decompressing. +UNCOMPRESSED=${UNCOMPRESSED:-false} + +# UGET_UPDATE can be set to true when the VERSION parameter of a program has +# been updated and you want µget to update all known variants (based on the +# checksums file) before installing the correct variant for the current system. +# When UGET_CHECKSUMS is not in use, this variable has no effect. +# Use UGET_UPDATE_ONLY if you want to just update the checksums without also +# installing the binary. +UGET_UPDATE=${UGET_UPDATE:-false} + +# UGET_UPDATE_ONLY is like UGET_UPDATE, but does not install the binary on +# the current system (i.e. it only touches the checksum file). +UGET_UPDATE_ONLY=${UGET_UPDATE_ONLY:-false} + +# UGET_DIRECTORY is where downloaded binaries will be placed. +UGET_DIRECTORY="${UGET_DIRECTORY:-_tools}" + +# UGET_CHECKSUMS is an optional path to a checksum file that µget should use to +# ensure subsequent downloads match previously known checksums to prevent +# tampering on the server side. +UGET_CHECKSUMS="${UGET_CHECKSUMS:-}" + +# UGET_VERSIONED_BINARIES can be set to true to append the binary's version to +# its filename, leading to final paths like "_tools/mytool-v1.2.3". This can +# help when µget is being used in Makefiles to improve staleness detection. +# Note that µget never deletes any installed binaries, so enabling this can +# lead to leftover binaries that you can cleanup at your own convenience. +UGET_VERSIONED_BINARIES=${UGET_VERSIONED_BINARIES:-false} + +# UGET_TEMPDIR is the root directory to use when creating new temporary dirs. +UGET_TEMPDIR="${UGET_TEMPDIR:-/tmp}" + +# UGET_PRINT_PATH can be set to "relative" to make µget only omit log output +# and only print the relative path to the binary, and set to "absolute" to +# output the absolute path. +UGET_PRINT_PATH="${UGET_PRINT_PATH:-no}" + +# UGET_HASHFUNC is the hashing function used to calculate file checksums. The +# output of this program is processed with awk to only print the first column. +UGET_HASHFUNC="${UGET_HASHFUNC:-sha256sum}" + +# UGET_GO_BUILD_CMD overwrites the default call to "go build" when installing +# a Go module. Use this to inject custom Go flags or toolchains. +# The given command is called with "-o BINARY_NAME MODULE_URL" with the pwd +# being inside a fake module that depends on the given module. +UGET_GO_BUILD_CMD="${UGET_GO_BUILD_CMD:-go build}" + +# UGET_GO_CHECKSUMS can be set to true to force checksums even for Go modules. +# This is disabled by default because the exact binaries being built depend on +# a lot of factors and usually it's a hassle to ensure everyone in a project +# has the *exact* same build environment. Go modules already make use of Google's +# GOSUMDB and should be "safe enough" by default with µget checksums. +UGET_GO_CHECKSUMS=${UGET_GO_CHECKSUMS:-false} + +############################################################################### +# Function library + +uget_mktemp() { + set -e + # --tmpdir does not work on MacOS + mktemp -d -p "$ABS_UGET_TEMPDIR" +} + +uget_log() { + if [ "$UGET_PRINT_PATH" = "no" ]; then + echo "$@" >&2 + fi +} + +uget_error() { + echo "$@" >&2 +} + +uget_lowercase() { + set -e + cat | tr '[:upper:]' '[:lower:]' +} + +uget_checksum_enabled() { + set -e + [ -n "$UGET_CHECKSUMS" ] +} + +uget_checksum_check() { + set -e + + local kvString="$1" + local downloadedBinary="$2" + + if ! uget_checksum_enabled; then return; fi + + local newChecksum + newChecksum="$(uget_checksum_calculate "$downloadedBinary")" + + local oldChecksum + oldChecksum="$(uget_checksum_read "$kvString")" + + if [ -n "$oldChecksum" ] && [ "$oldChecksum" != "$newChecksum" ]; then + uget_error + uget_error " *************************************************************************" + uget_error " SECURITY ERROR" + uget_error + uget_error " The downloaded file $downloadedBinary does not have the expected checksum." + uget_error + uget_error " Expected: $oldChecksum" + uget_error " Actual : $newChecksum" + uget_error + uget_error " If you are updating $IDENTIFIER, this error is expected." + uget_error " Re-run this command with the environment variable UGET_UPDATE=true to make" + uget_error " µget update the checksums for all known variants of $IDENTIFIER." + uget_error " Use UGET_UPDATE_ONLY=true if you want to just update the checksums and not" + uget_error " install the given binary on this machine." + uget_error " *************************************************************************" + uget_error + + return 1 + fi + + if [ -z "$oldChecksum" ]; then + uget_checksum_write "$kvString" "$newChecksum" + fi +} + +uget_checksum_read() { + set -e + + local kvString="$1" + + if [ -f "$UGET_CHECKSUMS" ]; then + awk -F'|' -v "binary=$IDENTIFIER" -v "kv=$kvString" '{ if ($1 == binary && $2 == kv) print $3 }' "$UGET_CHECKSUMS" + fi +} + +uget_checksum_calculate() { + set -e + "$UGET_HASHFUNC" "$1" | awk '{ print $1 }' +} + +uget_checksum_write() { + set -e + + local kvString="$1" + local checksum="$2" + + if [ -f "$UGET_CHECKSUMS" ]; then + local tempDir + tempDir="$(uget_mktemp)" + + # use awk to drop any existing hash for this binary/keyvalue combo + # (for better readability, do not invert the condition here); + # checking for NF (number of fields) to drop empty lines + awk \ + -F'|' -v "binary=$IDENTIFIER" -v "kv=$kvString" \ + '{ if (NF == 0 || ($1 == binary && $2 == kv)) {} else print }' \ + "$UGET_CHECKSUMS" > "$tempDir/checksums.txt" + + # add our new checksum + echo "$IDENTIFIER|$kvString|$checksum" >> "$tempDir/checksums.txt" + + # sort the file because it looks nicer and prevents ugly git diffs + sort "$tempDir/checksums.txt" > "$UGET_CHECKSUMS" + + rm -rf -- "$tempDir" + else + # start a new file + echo "$IDENTIFIER|$kvString|$checksum" > "$UGET_CHECKSUMS" + fi +} + +uget_url_placeholder() { + set -e + + case "$1" in + GOARCH) go env GOARCH ;; + GOOS) go env GOOS ;; + UARCH) uname -m | uget_lowercase ;; + UOS) uname -s | uget_lowercase ;; + *) uget_error "Unexpected placeholder $1."; return 1 ;; + esac +} + +# valueFromPair returns "foo" for "myvalue=foo;myothervalue=bar" when called with +# "myvalue" as the key. +uget_url_valueFromPair() { + set -e + + local kvString="$1" + local key="$2" + + # adding semicolons makes matching full keys easier + echo ";$kvString;" | sed -E "s/.*;$key=([^;]+).*/\\1/" +} + +uget_url_setKeyInPairs() { + set -e + + local kvString="$1" + local key="$2" + local value="$3" + + # first take the existing pairs and turn it into a multiline string; then + # awk out the existing pair for $key, + # then add a new pair and sort it all together, strip empty lines in case + # kvstring as empty, then join the multiple lines back into a single line + # and drop the trailing ';' + ( + echo "$kvString" | tr ';' "\n" | awk -F'=' -v "key=$key" '{ if ($1 != key) print }' + echo "$key=$value" + ) | sort | sed '/^[[:space:]]*$/d' | tr "\n" ';' | sed 's/;$//' +} + +uget_url_findPlaceholders() { + set -e + + # match all {...}, + # then sort and return only unique values (no need to replace the + # same placeholder multiple times) (this is important to allow for consistent + # matches when awk'ing through the checksum file), + # then remove braces and + # finally turn into a singleline string + echo "$1" | + grep -oE '\{[A-Z0-9_]+\}' | + sort -u | + tr -d '{}' | + awk '{printf("%s ", $0)}' +} + +# replaceLive() use live system-data to replace placeholders in the given pattern. +# It returns a string of form "KVSTRING|URL". +uget_url_replaceLive() { + set -e + + local urlPattern="$1" + local usedPlaceholders="" + + for placeholder in $(uget_url_findPlaceholders "$urlPattern"); do + # version is treated specially + [ "$placeholder" = "VERSION" ] && continue + + local replacement + replacement="$(uget_url_placeholder "$placeholder")" + urlPattern="$(echo "$urlPattern" | sed "s|{$placeholder}|$replacement|g")" + + # remember this placeholder and its value + usedPlaceholders="$usedPlaceholders;$placeholder=$replacement" + done + + # trim leading ";" + usedPlaceholders="$(echo "$usedPlaceholders" | sed 's/^;//')" + + echo "$usedPlaceholders|$urlPattern" +} + +# replaceWithArgs() does not ask the current system for the values when replacing +# a placeholder, but uses a given key-value pair string as the source. It also +# only returns the resulting string, since the used placeholders are known to +# the caller already. +uget_url_replaceWithArgs() { + set -e + + local urlPattern="$1" + local kvString="$2" + + for placeholder in $(uget_url_findPlaceholders "$urlPattern"); do + # version is treated specially + [ "$placeholder" = "VERSION" ] && continue + + local replacement + replacement="$(uget_url_valueFromPair "$kvString" "$placeholder")" + if [ -z "$replacement" ]; then + uget_error "Found no replacement string for placeholder $placeholder." + exit 1 + fi + + urlPattern="$(echo "$urlPattern" | sed "s|{$placeholder}|$replacement|g")" + done + + echo "$urlPattern" +} + +# returns "KVSTRING URL" +uget_url_build() { + set -e + + local kvString="${1:-}" + local pattern + + if [ -z "$kvString" ]; then + local result + result="$(uget_url_replaceLive "$URL_PATTERN")" + kvString="$(echo "$result" | cut -d'|' -f1)" + pattern="$(echo "$result" | cut -d'|' -f2)" + else + pattern="$(uget_url_replaceWithArgs "$URL_PATTERN" "$kvString")" + fi + + pattern="$(echo "$pattern" | sed "s|{VERSION}|$VERSION|g")" + + echo "$kvString|$pattern" +} + +# uget_http_download() uses either curl or wget to download the given URL into +# the current directory. +uget_http_download() { + set -e + + local url="$1" + + if command -v curl >/dev/null 2>&1; then + curl --fail -LO "$url" + elif command -v wget >/dev/null 2>&1; then + wget "$url" + else + uget_error "Neither curl nor wget are available." + return 1 + fi +} + +# uget_go_install creates a temporary Go module, adds the given URL as a +# dependency and then builds $BINARY. +uget_go_install() { + set -e + + local destinationDir="$1" + local kvString="$2" + local url="$3" + + # make sure GOARCH and GOOS will be set correctly + local os + local arch + os="$(uget_url_valueFromPair "$kvString" "GOOS")" + arch="$(uget_url_valueFromPair "$kvString" "GOARCH")" + + # since we crosscompile, we cannot do "GOBIN=(somewhere) go install url@version", + # because Go considers this to be dangerous behaviour: + # https://github.com/golang/go/issues/57485 + # Instead we create a dummy module and use the desired program as a dependency, + # *then* we're allowed to crosscompile it anywhere using "go build". Go figure. + + go mod init temp 2>/dev/null + go get "$url@$VERSION" + + local tmpFilename="__tmp.bin" + + # go build command is meant to be expanded + # shellcheck disable=SC2086 + GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" $UGET_GO_BUILD_CMD -o "$tmpFilename" "$url" + + mv "$tmpFilename" "$destinationDir/$BINARY" +} + +# uget_extract_archive extracts a downloaded archive and moves the one interesting +# file out of it. +uget_extract_archive() { + set -e + + local destinationDir="$1" + local archive="$2" + + if ! $UNCOMPRESSED; then + case "$archive" in + *.tar.gz | *.tgz) + tar xzf "$archive" + ;; + *.tar.bz2 | *.tbz2) + tar xjf "$archive" + ;; + *.tar.xz | *.txz) + tar xJf "$archive" + ;; + *.zip) + unzip "$archive" + ;; + *) + uget_error "Unknown file type: $archive" + return 1 + esac + fi + + # pattern is explicitly meant to be interpreted by the shell + # shellcheck disable=SC2086 + mv $BINARY_PATTERN "$destinationDir/$BINARY" + chmod +x "$destinationDir/$BINARY" +} + +# uget_download performs the actual download +# and places the binary in a given directory. +uget_download() { + set -e + + local destinationDir="$1" + local kvString="$2" + local url="$3" + + local startDir + startDir="$(pwd)" + + local tempDir + tempDir="$(uget_mktemp)" + + cd "$tempDir" + + if $GO_MODULE; then + uget_go_install "$destinationDir" "$kvString" "$url" + else + uget_http_download "$url" + + local archive + archive="$(ls)" + + uget_extract_archive "$destinationDir" "$archive" + fi + + cd "$startDir" + rm -rf -- "$tempDir" +} + +# ready() checks if the desired binary already exists in the desired version. +uget_ready() { + local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" + if ! [ -f "$fullFinalPath" ]; then + return 1 + fi + + # skip further checks if we're using versioned binaries + if $UGET_VERSIONED_BINARIES; then + return 0 + fi + + local versionFile="$fullFinalPath.version" + [ -f "$versionFile" ] && [ "$VERSION" = "$(cat "$versionFile")" ] +} + +# install() downloads the binary, checks the checksum and places it in UGET_DIRECTORY. +uget_install() { + set -e + + local kvString="$1" + local url="$2" + local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" + local versionFile="$fullFinalPath.version" + + local startDir + startDir="$(pwd)" + + local tempDir + tempDir="$(uget_mktemp)" + + uget_log "Downloading $IDENTIFIER version $VERSION ..." + uget_download "$tempDir" "$kvString" "$url" + + local fullTempBinary="$tempDir/$BINARY" + uget_checksum_check "$kvString" "$fullTempBinary" + + # if everything is fine, place the binary in its final location + mv "$fullTempBinary" "$fullFinalPath" + + if ! $UGET_VERSIONED_BINARIES; then + echo "$VERSION" > "$versionFile" + fi + + uget_log "Installed at $UGET_DIRECTORY/$BINARY." + + cd "$startDir" + rm -rf -- "$tempDir" +} + +# update() downloads the binary, updates the checksums and discards the binary. +uget_update() { + set -e + + local kvString="$1" + local url="$2" + + local startDir + startDir="$(pwd)" + + local tempDir + tempDir="$(uget_mktemp)" + + uget_log " ~> $kvString" + uget_download "$tempDir" "$kvString" "$url" + + local fullBinaryPath="$tempDir/$BINARY" + + local checksum + checksum="$(uget_checksum_calculate "$fullBinaryPath")" + + uget_checksum_write "$kvString" "$checksum" + + cd "$startDir" + rm -rf -- "$tempDir" +} + +############################################################################### +# General Setup Logic + +# get CLI flags +export URL_PATTERN="$1" +export IDENTIFIER="$2" +export VERSION="$3" +BINARY_PATTERN="${4:-**/$IDENTIFIER}" + +export BINARY="$IDENTIFIER" +if $UGET_VERSIONED_BINARIES; then + BINARY="$IDENTIFIER-$VERSION" +fi + +# ensure target directory exists +mkdir -p "$UGET_DIRECTORY" "$UGET_TEMPDIR" + +ABS_UGET_DIRECTORY="$(realpath "$UGET_DIRECTORY")" +ABS_UGET_TEMPDIR="$(realpath "$UGET_TEMPDIR")" + +if $GO_MODULE && ! $UGET_GO_CHECKSUMS; then + UGET_CHECKSUMS="" +fi + +if uget_checksum_enabled; then + touch "$UGET_CHECKSUMS" + UGET_CHECKSUMS="$(realpath "$UGET_CHECKSUMS")" +else + if $GO_MODULE; then + if $UGET_UPDATE || $UGET_UPDATE_ONLY; then + uget_error "Checksums are disabled for Go modules, cannot update $IDENTIFIER checksums." + # This is not an error because in complex Makefiles, there might be 3 binaries required + # for one make target, and if one of them is a Go module and you have no direct way + # to just update a single binary, then running "UGET_UPDATE make complex-task" would + # fail at the Go module. It's simply more convenient to just warn and move on to the next + # binary. + fi + fi + + UGET_UPDATE=false + UGET_UPDATE_ONLY=false +fi + +############################################################################### +# Main application logic + +# When in update-only mode, we do not download the binary for the current system +# only, but instead for all known variants based on the checksums file, +# recalculate the checksums and then discard the temporary binaries. Otherwise +# after updating the checksums we continue with the regular install logic. + +if $UGET_UPDATE || $UGET_UPDATE_ONLY; then + uget_log "Updating checksums for $IDENTIFIER ..." + + # Find and process all known variants... + awk -F'|' -v "binary=$IDENTIFIER" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS" | while IFS= read -r kvString; do + result="$(uget_url_build "$kvString")" + url="$(echo "$result" | cut -d'|' -f2)" + + # download binary into tempdir, update checksums, but then delete it again + uget_update "$kvString" "$url" + done + + uget_log "All checksums were updated." +fi + +if ! $UGET_UPDATE_ONLY; then + if ! uget_ready; then + # Replace placeholders in the URL with system-specific data, like arch or OS + result="$(uget_url_build)" + kvString="$(echo "$result" | cut -d'|' -f1)" + url="$(echo "$result" | cut -d'|' -f2)" + + # Go modules usually do not have arch/os infos in their module names, but + # the resulting binary still depends on the local system. To support checksums + # for different machines, we always assume GOARCH and GOOS are "used placeholders". + if $GO_MODULE; then + kvString="$(uget_url_setKeyInPairs "$kvString" "GOARCH" "$(uget_url_placeholder GOARCH)")" + kvString="$(uget_url_setKeyInPairs "$kvString" "GOOS" "$(uget_url_placeholder GOOS)")" + fi + + uget_install "$kvString" "$url" + fi + + case "$UGET_PRINT_PATH" in + absolute) echo "$ABS_UGET_DIRECTORY/$BINARY" ;; + relative) echo "$UGET_DIRECTORY/$BINARY" ;; + esac +fi diff --git a/staging/src/github.com/kcp-dev/client-go/Makefile b/staging/src/github.com/kcp-dev/client-go/Makefile index 43de53cf432..50546b248da 100644 --- a/staging/src/github.com/kcp-dev/client-go/Makefile +++ b/staging/src/github.com/kcp-dev/client-go/Makefile @@ -15,24 +15,20 @@ # We need bash for some conditional logic below. SHELL := /usr/bin/env bash -e -KCP_ROOT_DIR ?= $(abspath ../../../../..) +KCP_ROOT_PATH = ../../../../.. +KCP_ROOT_DIR ?= $(abspath $(KCP_ROOT_PATH)) -GO_INSTALL = $(KCP_ROOT_DIR)/hack/go-install.sh - -TOOLS_DIR=hack/tools +TOOLS_DIR = hack/tools TOOLS_GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -GOBIN_DIR=$(abspath ./bin) -PATH := $(GOBIN_DIR):$(TOOLS_GOBIN_DIR):$(PATH) +PATH := $(TOOLS_GOBIN_DIR):$(PATH) -GOLANGCI_LINT_VER := v2.1.6 -GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) -GOLANGCI_LINT_FLAGS ?= +GOLANGCI_LINT_VER = $(shell $(MAKE) --no-print-directory -C $(KCP_ROOT_PATH) golangci-lint-version) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VER) $(GOLANGCI_LINT): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/v2/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + $(MAKE) -C $(KCP_ROOT_PATH) $(GOLANGCI_LINT) -tools: $(CODE_GENERATOR) +tools: $(CODE_GENERATOR) $(GOLANGCI_LINT) .PHONY: tools codegen: diff --git a/staging/src/github.com/kcp-dev/code-generator/Makefile b/staging/src/github.com/kcp-dev/code-generator/Makefile index 993687697e5..caf043f43c4 100644 --- a/staging/src/github.com/kcp-dev/code-generator/Makefile +++ b/staging/src/github.com/kcp-dev/code-generator/Makefile @@ -14,24 +14,21 @@ SHELL := /usr/bin/env bash -KCP_ROOT_DIR ?= $(abspath ../../../../..) - -GO_INSTALL = $(KCP_ROOT_DIR)/hack/go-install.sh +KCP_ROOT_PATH = ../../../../.. +KCP_ROOT_DIR ?= $(abspath $(KCP_ROOT_PATH)) BUILD_DEST ?= _build BUILDFLAGS ?= CMD ?= $(notdir $(wildcard ./cmd/*)) -TOOLS_DIR=hack/tools -GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -TMPDIR := $(shell mktemp -d) +TOOLS_DIR = hack/tools +TOOLS_GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -GOLANGCI_LINT_VER := v2.1.6 -GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) +GOLANGCI_LINT_VER = $(shell $(MAKE) --no-print-directory -C $(KCP_ROOT_PATH) golangci-lint-version) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VER) $(GOLANGCI_LINT): - GOBIN=$(GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + $(MAKE) -C $(KCP_ROOT_PATH) $(GOLANGCI_LINT) .PHONY: imports imports: WHAT ?= diff --git a/test/e2e/authorizer/utils.go b/test/e2e/authorizer/utils.go index 8af1c531f93..e82b8dca56e 100644 --- a/test/e2e/authorizer/utils.go +++ b/test/e2e/authorizer/utils.go @@ -44,7 +44,13 @@ func RunWebhook(ctx context.Context, t *testing.T, port string, response string) "--listen", address, } - cmd := exec.CommandContext(ctx, "httest", args...) + // get the binary name from our Makefile + httestBinary := os.Getenv("HTTEST") + if httestBinary == "" { + httestBinary = "httest" + } + + cmd := exec.CommandContext(ctx, httestBinary, args...) if err := cmd.Start(); err != nil { cancel() t.Fatalf("Failed to start webhook: %v", err)