From aae201d0c4a1ecfea68919ea00204b902faf2ef4 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 31 Oct 2025 19:55:55 +0100 Subject: [PATCH 1/9] use a script to download premade binaries instead of compiling them On-behalf-of: @SAP christoph.mewes@sap.com --- Makefile | 53 +++-- hack/go-install.sh | 62 ------ hack/tools.checksums | 15 ++ hack/uget.sh | 497 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 545 insertions(+), 82 deletions(-) delete mode 100755 hack/go-install.sh create mode 100644 hack/tools.checksums create mode 100755 hack/uget.sh diff --git a/Makefile b/Makefile index 6add1025f70..b0500ce9ed1 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ ifeq ($(CI),true) $(shell git config --global --add safe.directory '*') endif -GO_INSTALL = ./hack/go-install.sh - -TOOLS_DIR=hack/tools +TOOLS_DIR = hack/tools +export UGET_DIRECTORY = $(TOOLS_DIR) +export UGET_CHECKSUMS = hack/tools.checksums ROOT_DIR=$(abspath .) TOOLS_GOBIN_DIR := $(abspath $(TOOLS_DIR)) GOBIN_DIR=$(abspath ./bin) @@ -42,30 +42,30 @@ endif CONTROLLER_GEN_VER := v0.17.3 CONTROLLER_GEN_BIN := controller-gen -CONTROLLER_GEN := $(TOOLS_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER) +CONTROLLER_GEN := $(TOOLS_DIR)/$(CONTROLLER_GEN_BIN) export CONTROLLER_GEN # so hack scripts can use it YAML_PATCH_VER ?= v0.0.11 YAML_PATCH_BIN := yaml-patch -YAML_PATCH := $(TOOLS_DIR)/$(YAML_PATCH_BIN)-$(YAML_PATCH_VER) +YAML_PATCH := $(TOOLS_DIR)/$(YAML_PATCH_BIN) export YAML_PATCH # so hack scripts can use it -GOLANGCI_LINT_VER := v2.1.6 +GOLANGCI_LINT_VER := 2.1.6 GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN) GOLANGCI_LINT_FLAGS ?= -HTTEST_VER := v0.3.2 +HTTEST_VER := 0.3.4 HTTEST_BIN := httest -HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) +HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN) -GOTESTSUM_VER := v1.12.3 +GOTESTSUM_VER := 1.12.3 GOTESTSUM_BIN := gotestsum -GOTESTSUM := $(abspath $(TOOLS_DIR))/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER) +GOTESTSUM := $(abspath $(TOOLS_DIR))/$(GOTESTSUM_BIN) -LOGCHECK_VER := v0.9.0 +LOGCHECK_VER := d35c84c015fe03a1421e5f2ce1e3c0c3bc38d077 LOGCHECK_BIN := logcheck -LOGCHECK := $(TOOLS_GOBIN_DIR)/$(LOGCHECK_BIN)-$(LOGCHECK_VER) +LOGCHECK := $(TOOLS_GOBIN_DIR)/$(LOGCHECK_BIN) export LOGCHECK # so hack scripts can use it KCP_APIGEN_BIN := apigen @@ -129,13 +129,22 @@ install: require-jq require-go require-git verify-go-versions ## Install the pro .PHONY: install $(GOLANGCI_LINT): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/v2/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + @hack/uget.sh \ + https://github.com/golangci/golangci-lint/releases/download/v{VERSION}/golangci-lint-{VERSION}-{GOOS}-{GOARCH}.tar.gz \ + ${GOLANGCI_LINT_BIN} \ + ${GOLANGCI_LINT_VER} $(HTTEST): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) go.xrstf.de/httest $(HTTEST_BIN) $(HTTEST_VER) + @hack/uget.sh \ + https://codeberg.org/xrstf/httest/releases/download/v{VERSION}/httest_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ + ${HTTEST_BIN} \ + ${HTTEST_VER} $(LOGCHECK): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) sigs.k8s.io/logtools/logcheck $(LOGCHECK_BIN) $(LOGCHECK_VER) + @GO_MODULE=true hack/uget.sh \ + sigs.k8s.io/logtools/logcheck \ + ${LOGCHECK_BIN} \ + $(LOGCHECK_VER) $(KCP_APIGEN_GEN): pushd . && cd staging/src/github.com/kcp-dev/sdk && GOBIN=$(TOOLS_GOBIN_DIR) go install ./cmd/apigen && popd @@ -197,13 +206,17 @@ tools: $(GOLANGCI_LINT) $(HTTEST) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PAT .PHONY: tools $(CONTROLLER_GEN): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) sigs.k8s.io/controller-tools/cmd/controller-gen $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER) + @GO_MODULE=true hack/uget.sh sigs.k8s.io/controller-tools/cmd/controller-gen ${CONTROLLER_GEN_BIN} $(CONTROLLER_GEN_VER) $(YAML_PATCH): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) + @GO_MODULE=true hack/uget.sh github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) $(GOTESTSUM): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) gotest.tools/gotestsum $(GOTESTSUM_BIN) $(GOTESTSUM_VER) + @hack/uget.sh \ + https://github.com/gotestyourself/gotestsum/releases/download/v{VERSION}/gotestsum_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ + ${GOTESTSUM_BIN} \ + ${GOTESTSUM_VER} \ + ${GOTESTSUM_BIN} crds: $(CONTROLLER_GEN) $(YAML_PATCH) ## Generate crds ./hack/update-codegen-crds.sh @@ -267,7 +280,7 @@ BOILERPLATE_KUBERNETES_FILES := $(abspath ./hack/boilerplate/boilerplate_kuberne .PHONY: verify-boilerplate verify-boilerplate: ## Verify boilerplate - hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate --skip docs/venv --skip pkg/network/dialer --skip-files-list $(BOILERPLATE_MODIFIED_FILES) --skip-files-list $(BOILERPLATE_KUBERNETES_FILES) + hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate --skip hack/uget.sh --skip docs/venv --skip pkg/network/dialer --skip-files-list $(BOILERPLATE_MODIFIED_FILES) --skip-files-list $(BOILERPLATE_KUBERNETES_FILES) hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate/boilerplate_modified --filenames-list $(BOILERPLATE_MODIFIED_FILES) hack/verify_boilerplate.py --boilerplate-dir=hack/boilerplate/boilerplate_kubernetes --filenames-list $(BOILERPLATE_KUBERNETES_FILES) diff --git a/hack/go-install.sh b/hack/go-install.sh deleted file mode 100755 index ef6f8af6345..00000000000 --- a/hack/go-install.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The KCP Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Originally copied from -# https://github.com/kubernetes-sigs/cluster-api-provider-gcp/blob/c26a68b23e9317323d5d37660fe9d29b3d2ff40c/scripts/go_install.sh - -set -o errexit -set -o nounset -set -o pipefail - -if [[ -z "${1:-}" ]]; then - echo "must provide module as first parameter" - exit 1 -fi - -if [[ -z "${2:-}" ]]; then - echo "must provide binary name as second parameter" - exit 1 -fi - -if [[ -z "${3:-}" ]]; then - echo "must provide version as third parameter" - exit 1 -fi - -if [[ -z "${GOBIN:-}" ]]; then - echo "GOBIN is not set. Must set GOBIN to install the bin in a specified directory." - exit 1 -fi - -mkdir -p "${GOBIN}" - -tmp_dir=$(mktemp -d -t goinstall_XXXXXXXXXX) -function clean { - rm -rf "${tmp_dir}" -} -trap clean EXIT - -rm "${GOBIN}/${2}"* > /dev/null 2>&1 || true - -cd "${tmp_dir}" - -# create a new module in the tmp directory -go mod init fake/mod - -# install the golang module specified as the first argument -go install -tags kcptools "${1}@${3}" -mv "${GOBIN}/${2}" "${GOBIN}/${2}-${3}" -ln -sf "${GOBIN}/${2}-${3}" "${GOBIN}/${2}" \ No newline at end of file diff --git a/hack/tools.checksums b/hack/tools.checksums new file mode 100644 index 00000000000..108f1f5e02c --- /dev/null +++ b/hack/tools.checksums @@ -0,0 +1,15 @@ +controller-gen|GOARCH=amd64;GOOS=linux|cc3588262701701c3676fdf11094ccb5633650a2fc2cab9b237a0ea83214a75e +controller-gen|GOARCH=arm64;GOOS=darwin|e44b122bfc0911d98db7d5f6129c08dfd3df4da1dc8692c4aef4a8bd903edfcf +controller-gen|GOARCH=arm64;GOOS=linux|3514501d47db429950feaea34836ed59813c1f8421fbbbb8217884eb125c0cf7 +golangci-lint|GOARCH=amd64;GOOS=linux|7009324a8aad93c1f84dce1c3cf61181bdd6b68e5f1b8b5d6971662258255050 +golangci-lint|GOARCH=arm64;GOOS=darwin|96dc1dbbb686fc7982b9ea4b800f3d234a2d9fb78b0344f287e8d28ee11bc72b +golangci-lint|GOARCH=arm64;GOOS=linux|c51ff5b21be688b043baea44de7dd855cf07b855c14f0de405bfaf922b1d7634 +gotestsum|GOARCH=amd64;GOOS=linux|2e505a9368568aa7422e0a90ef77acc8807c0d3272ab81c7a69e3e8688d1cf65 +gotestsum|GOARCH=arm64;GOOS=darwin|020be8d14358c7ac4155e296436057cf4b1f1232f8f8f3d71f22a0e7a5504340 +gotestsum|GOARCH=arm64;GOOS=linux|2f8517768c2831750cb372e379404a059dbd20f2b1f79bcc235c4cab4540cb10 +httest|GOARCH=amd64;GOOS=linux|d524eea4fa0bcb9376466a6da7892d88556ed6c03d70f282946e5ca8474b4232 +httest|GOARCH=arm64;GOOS=darwin|12ac43f851459bc1e82302ff2d2bde987970c2c0a47933f79ddd3c277bf4a93d +httest|GOARCH=arm64;GOOS=linux|6f317e0e9857a7582bbf881d30c8c64ead353312d3141b31cf1261de002d91ea +yaml-patch|GOARCH=amd64;GOOS=linux|6eae938136a18a5730700676f89ce7a0d4a89d332219fad180202b329501a88b +yaml-patch|GOARCH=arm64;GOOS=darwin|ce2f9448472b0e69e44c28c7e6879caae46664aa61e251fe7f181b78158ff9b0 +yaml-patch|GOARCH=arm64;GOOS=linux|f379170ba41d9d02446cbdf42fd48b47892a23f9333d9385ad8306449fc22d8b diff --git a/hack/uget.sh b/hack/uget.sh new file mode 100755 index 00000000000..f90e5739e93 --- /dev/null +++ b/hack/uget.sh @@ -0,0 +1,497 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget +# SPDX-License-Identifier: MIT +# +# µget 0.2.0 – your friendly downloader +# ------------------------------------- +# +# µget can download software as binaries, archives or Go modules. +# +# Usage: ./uget.sh URL_PATTERN VERSION +# +# µget supports a large range of environment variables to customize its +# behaviour. + +set -euo pipefail + +############################################################################### +# Configuration + +# UGET_UPDATE can be set to true when the VERSION parameter of a program has +# been updated and you want µget to update all known variants (based on the +# checksums file). When UGET_CHECKSUMS is not in use, this variable has no +# effect. +UGET_UPDATE=${UGET_UPDATE:-false} + +# UGET_DIRECTORY is where downloaded binaries will be placed. +UGET_DIRECTORY="${UGET_DIRECTORY:-_tools}" + +# UGET_CHECKSUMS is an optional path to a checksum file that µget should use to +# ensure subsequent downloads match previously known checksums to prevent +# tampering on the server side. +UGET_CHECKSUMS="${UGET_CHECKSUMS:-}" + +# UGET_TEMPDIR is the root directory to use when creating new temporary dirs. +UGET_TEMPDIR="${UGET_TEMPDIR:-/tmp}" + +# UGET_PRINT_PATH can be set to "relative" to make µget only omit log output +# and only print the relative path to the binary, and set to "absolute" to +# output the absolute path. +UGET_PRINT_PATH="${UGET_PRINT_PATH:-no}" + +# UGET_HASHFUNC is the hashing function used to calculate file checksums. The +# output of this program is processed with awk to only print the first column. +UGET_HASHFUNC="${UGET_HASHFUNC:-sha256sum}" + +# UGET_GO_BUILD_CMD overwrites the default call to "go build" when installing +# a Go module. Use this to inject custom Go flags or toolchains. +# The given command is called with "-o BINARY_NAME MODULE_URL" with the pwd +# being inside a fake module that depends on the given module. +UGET_GO_BUILD_CMD="${UGET_GO_BUILD_CMD:-go build}" + +# UGET_GO_CHECKSUMS can be set to true to force checksums even for Go modules. +# This is disabled by default because the exact binaries being built depend on +# a lot of factors and usually it's a hassle to ensure everyone in a project +# has the *exact* same build environment. Go modules already make use of Google's +# GOSUMDB and should be "safe enough" by default with µget checksums. +UGET_GO_CHECKSUMS=${UGET_GO_CHECKSUMS:-false} + +############################################################################### +# Function library + +uget::mktemp() { + # --tmpdir does not work on MacOS + mktemp -d -p "$ABS_UGET_TEMPDIR" +} + +uget::log() { + if [[ "$UGET_PRINT_PATH" == "no" ]]; then + echo "$@" >&2 + fi +} + +uget::error() { + echo "$@" >&2 +} + +uget::lowercase() { + cat | tr '[:upper:]' '[:lower:]' +} + +uget::checksum::enabled() { + [[ -n "$UGET_CHECKSUMS" ]] +} + +uget::checksum::check() { + local kvString="$1" + local downloadedBinary="$2" + + if ! uget::checksum::enabled; then return; fi + + local newChecksum + newChecksum="$(uget::checksum::calculate "$downloadedBinary")" + + local oldChecksum + oldChecksum="$(uget::checksum::read "$kvString")" + + if [[ -n "$oldChecksum" ]] && [[ "$oldChecksum" != "$newChecksum" ]]; then + uget::error + uget::error " *************************************************************************" + uget::error " SECURITY ERROR" + uget::error + uget::error " The downloaded file $downloadedBinary does not have the expected checksum." + uget::error + uget::error " Expected: $oldChecksum" + uget::error " Actual : $newChecksum" + uget::error + uget::error " If you are updating $BINARY, this error is expected." + uget::error " Re-run this command with the environment variable UGET_UPDATE=true to make" + uget::error " µget update the checksums for all known variants of $BINARY." + uget::error " *************************************************************************" + uget::error + + return 1 + fi + + uget::checksum::write "$kvString" "$newChecksum" +} + +uget::checksum::read() { + local kvString="$1" + + if [[ -f "$UGET_CHECKSUMS" ]]; then + awk -F'|' "{ if (\$1 == \"$BINARY\" && \$2 == \"$kvString\") print \$3 }" "$UGET_CHECKSUMS" + fi +} + +uget::checksum::calculate() { + "$UGET_HASHFUNC" "$1" | awk '{ print $1 }' +} + +uget::checksum::write() { + local kvString="$1" + local checksum="$2" + + if [[ -f "$UGET_CHECKSUMS" ]]; then + local tempDir + tempDir="$(uget::mktemp)" + + # use awk to drop any existing hash for this binary/keyvalue combo + # (for better readability, do not invert the condition here); + # grep will drop any empty lines + awk \ + -F'|' \ + "{ if (\$1 == \"$BINARY\" && \$2 == \"$kvString\") {} else print }" \ + "$UGET_CHECKSUMS" | grep . > "$tempDir/checksums.txt" + + # add our new checksum + echo "$BINARY|$kvString|$checksum" >> "$tempDir/checksums.txt" + + # sort the file because it looks nicer and prevents ugly git diffs + cat "$tempDir/checksums.txt" | sort > "$UGET_CHECKSUMS" + + rm -rf -- "$tempDir" + else + # start a new file + echo "$BINARY|$kvString|$checksum" >> "$UGET_CHECKSUMS" + fi +} + +uget::url::placeholder() { + case "$1" in + GOARCH) go env GOARCH ;; + GOOS) go env GOOS ;; + UARCH) uname -m | uget::lowercase ;; + UOS) uname -s | uget::lowercase ;; + *) uget::error "Unexpected placeholder $1."; return 1 ;; + esac +} + +# valueFromPair returns "foo" for ";myvalue=foo;myothervalue=bar;" when called with +# "myvalue" as the key. The kv string must already be surrounded with semicolons. +uget::url::valueFromPair() { + local kvString="$1" + local key="$2" + + (echo "$kvString" | grep -oE ";$key=([^;]+)" || true) | cut -f2 -d'=' +} + +uget::url::setKeyInPairs() { + local kvString="$1" + local key="$2" + local value="$3" + + # first take the existing pairs but without the one for the given key, + # then add a new pair and sort it all together, strip empty lines in case + # kvstring as empty, then join the multiple lines back into a single line + # and drop the trailing ';' + ( + echo "$kvString" | tr ';' "\n" | (grep -vE "^$key=" || true) + echo "$key=$value" + ) | sort | sed '/^[[:space:]]*$/d' | tr "\n" ';' | sed 's/;$//' +} + +uget::url::findPlaceholders() { + # match all {...}, + # then sort and return only unique values (no need to replace the + # same placeholder multiple times) (this is important to allow for consistent + # matches when awk'ing through the checksum file), + # then remove braces and + # finally turn into a singleline string + echo "$1" | + grep -oE '\{[A-Z0-9_]+\}' | + sort -u | + tr -d '{}' | + awk '{printf("%s ", $0)}' +} + +# replaceLive() use live system-data to replace placeholders in the given pattern. +# It returns a string of form "KVSTRING|URL". +uget::url::replaceLive() { + local urlPattern="$1" + local usedPlaceholders="" + + for placeholder in $(uget::url::findPlaceholders "$urlPattern"); do + # version is treated specially + [[ "$placeholder" == "VERSION" ]] && continue + + replacement="$(uget::url::placeholder "$placeholder")" + urlPattern="${urlPattern//\{$placeholder\}/$replacement}" + + # remember this placeholder and its value + usedPlaceholders="$usedPlaceholders;$placeholder=$replacement" + done + + # trim leading ";" (this is safe even for zero-length strings) + usedPlaceholders="${usedPlaceholders:1}" + + echo "$usedPlaceholders|$urlPattern" +} + +# replaceWithArgs() does not ask the current system for the values when replacing +# a placeholder, but uses a given key-value pair string as the source. It also +# only returns the resulting string, since the used placeholders are known to +# the caller already. +uget::url::replaceWithArgs() { + local urlPattern="$1" + local kvString="$2" + + # make matching easier + kvString=";$kvString;" + + for placeholder in $(uget::url::findPlaceholders "$urlPattern"); do + # version is treated specially + [[ "$placeholder" == "VERSION" ]] && continue + + replacement="$(uget::url::valueFromPair "$kvString" "$placeholder")" + if [[ -z "$replacement" ]]; then + uget::error "Found no replacement string for placeholder $placeholder." + exit 1 + fi + + urlPattern="${urlPattern//\{$placeholder\}/$replacement}" + done + + echo "$urlPattern" +} + +# returns "KVSTRING URL" +uget::url::build() { + local kvString="${1:-}" + local pattern + + if [[ -z "$kvString" ]]; then + result="$(uget::url::replaceLive "$URL_PATTERN")" + kvString="$(echo "$result" | cut -d'|' -f1)" + pattern="$(echo "$result" | cut -d'|' -f2)" + else + pattern="$(uget::url::replaceWithArgs "$URL_PATTERN" "$kvString")" + fi + + pattern="${pattern//\{VERSION\}/$VERSION}" + + echo "$kvString|$pattern" +} + +# uget::fetch() detects whether wget or curl is available for downloading something. +uget::fetch() { + local url="$1" + + if command -v curl &> /dev/null; then + curl --fail -LO "$url" + elif command -v wget &> /dev/null; then + wget "$url" + else + uget::error "Neither curl nor wget are available." + return 1 + fi +} + +# uget::download performs the actual download +# and places the binary in a given directory. +uget::download() { + local destinationDir="$1" + local kvString="$2" + local url="$3" + + local startDir + startDir="$(pwd)" + + local tempDir + tempDir="$(uget::mktemp)" + + cd "$tempDir" + + if $GO_MODULE; then + # make sure GOARCH and GOOS are set correctly + os="$(uget::url::valueFromPair ";$kvString;" "GOOS")" + arch="$(uget::url::valueFromPair ";$kvString;" "GOARCH")" + + # since we crosscompile, we cannot do "GOBIN=(somewhere) go install url@version", + # because Go considers this to be dangerous behaviour: + # https://github.com/golang/go/issues/57485 + # Instead we create a dummy module and use the desired program as a dependency, + # *then* we're allowed to crosscompile it anywhere using "go build". Go figure. + + go mod init temp 2>/dev/null + go get "$url@$VERSION" + + GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" GOBIN=$(realpath .) $UGET_GO_BUILD_CMD -o "$BINARY" "$url" + + mv "$BINARY" "$destinationDir/$BINARY" + else + uget::fetch "$url" + archive="$(ls)" + + if ! $UNCOMPRESSED; then + case "$archive" in + *.tar.gz | *.tgz) + tar xzf "$archive" + ;; + *.tar.bz2 | *.tbz2) + tar xjf "$archive" + ;; + *.tar.xz | *.txz) + tar xJf "$archive" + ;; + *.zip) + unzip "$archive" + ;; + *) + uget::error "Unknown file type: $archive" + return 1 + esac + fi + + # pattern is explicitly meant to be interpreted by the shell + # shellcheck disable=SC2086 + mv $BINARY_PATTERN "$destinationDir/$BINARY" + chmod +x "$destinationDir/$BINARY" + fi + + cd "$startDir" + rm -rf -- "$tempDir" +} + +# ready() checks if the desired binary already exists in the desired version. +uget::ready() { + local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" + local versionFile="$fullFinalPath.version" + + [ -f "$fullFinalPath" ] && [ -f "$versionFile" ] && [ "$VERSION" == "$(cat "$versionFile")" ] +} + +# install() downloads the binary, checks the checksum and places it in UGET_DIRECTORY. +uget::install() { + local kvString="$1" + local url="$2" + local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" + local versionFile="$fullFinalPath.version" + + local startDir + startDir="$(pwd)" + + local tempDir + tempDir="$(uget::mktemp)" + + uget::log "Downloading $BINARY version $VERSION ..." + uget::download "$tempDir" "$kvString" "$url" + + fullBinaryPath="$tempDir/$BINARY" + uget::checksum::check "$kvString" "$fullBinaryPath" + + # if everything is fine, place the binary in its final location + mv "$fullBinaryPath" "$fullFinalPath" + echo -n "$VERSION" > "$versionFile" + + uget::log "Installed at $UGET_DIRECTORY/$BINARY." + + cd "$startDir" + rm -rf -- "$tempDir" +} + +# update() downloads the binary, updates the checksums and discards the binary. +uget::update() { + local kvString="$1" + local url="$2" + + local startDir + startDir="$(pwd)" + + local tempDir + tempDir="$(uget::mktemp)" + + uget::log " ~> $kvString" + uget::download "$tempDir" "$kvString" "$url" + + fullBinaryPath="$tempDir/$BINARY" + + local checksum + checksum="$(uget::checksum::calculate "$fullBinaryPath")" + + uget::checksum::write "$kvString" "$checksum" + + cd "$startDir" + rm -rf -- "$tempDir" +} + +############################################################################### +# General Setup Logic + +# get CLI flags +export URL_PATTERN="$1" +export BINARY="$2" +export VERSION="$3" +BINARY_PATTERN="${4:-**/$BINARY}" + +# additional per-tool configuration, which is not prefixed with UGET_ because +# it's scoped to single tools only +GO_MODULE=${GO_MODULE:-false} +UNCOMPRESSED=${UNCOMPRESSED:-false} + +# ensure target directory exists +mkdir -p "$UGET_DIRECTORY" "$UGET_TEMPDIR" + +ABS_UGET_DIRECTORY="$(realpath "$UGET_DIRECTORY")" +ABS_UGET_TEMPDIR="$(realpath "$UGET_TEMPDIR")" + +if $GO_MODULE && ! $UGET_GO_CHECKSUMS; then + UGET_CHECKSUMS="" +fi + +if uget::checksum::enabled; then + touch "$UGET_CHECKSUMS" + UGET_CHECKSUMS="$(realpath "$UGET_CHECKSUMS")" +else + if $UGET_UPDATE && $GO_MODULE; then + uget::error "Checksums are disabled for Go modules, cannot update them." + exit 1 + fi + + UGET_UPDATE=false +fi + +############################################################################### +# Main application logic + +# When in update mode, we do not download the binary for the current system +# only, but instead for all known variants based on the checksums file, +# recalculate the checksums and then discard the temporary binaries. + +if $UGET_UPDATE; then + uget::log "Updating checksums for $BINARY ..." + + # Find and process all known variants... + while read -r kvString; do + result="$(uget::url::build "$kvString")" + url="$(echo "$result" | cut -d'|' -f2)" + + # download binary into tempdir, update checksums, but then delete it again + uget::update "$kvString" "$url" + done < <(awk -F'|' "{ if (\$1 == \"$BINARY\") print \$2 }" "$UGET_CHECKSUMS") + + uget::log "All checksums were updated." +else + if ! uget::ready; then + # Replace placeholders in the URL with system-specific data, like arch or OS + result="$(uget::url::build)" + kvString="$(echo "$result" | cut -d'|' -f1)" + url="$(echo "$result" | cut -d'|' -f2)" + + # Go modules usually do not have arch/os infos in their module names, but + # the resulting binary still depends on the local system. To support checksums + # for different machines, we always assume GOARCH and GOOS are "used placeholders". + if $GO_MODULE; then + kvString="$(uget::url::setKeyInPairs "$kvString" "GOARCH" "$(go env GOARCH)")" + kvString="$(uget::url::setKeyInPairs "$kvString" "GOOS" "$(go env GOOS)")" + fi + + uget::install "$kvString" "$url" + fi + + case "$UGET_PRINT_PATH" in + absolute) echo "$ABS_UGET_DIRECTORY/$BINARY" ;; + relative) echo "$UGET_DIRECTORY/$BINARY" ;; + esac +fi From 5a02126169331063f7b900b02025a2cd4ff053f7 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 31 Oct 2025 22:44:43 +0100 Subject: [PATCH 2/9] improve staging modules On-behalf-of: @SAP christoph.mewes@sap.com --- .../src/github.com/kcp-dev/client-go/Makefile | 16 +++++----------- .../github.com/kcp-dev/code-generator/Makefile | 13 ++++--------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/staging/src/github.com/kcp-dev/client-go/Makefile b/staging/src/github.com/kcp-dev/client-go/Makefile index 43de53cf432..9ad80ea2da7 100644 --- a/staging/src/github.com/kcp-dev/client-go/Makefile +++ b/staging/src/github.com/kcp-dev/client-go/Makefile @@ -17,22 +17,16 @@ SHELL := /usr/bin/env bash -e KCP_ROOT_DIR ?= $(abspath ../../../../..) -GO_INSTALL = $(KCP_ROOT_DIR)/hack/go-install.sh - -TOOLS_DIR=hack/tools +TOOLS_DIR = hack/tools TOOLS_GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -GOBIN_DIR=$(abspath ./bin) -PATH := $(GOBIN_DIR):$(TOOLS_GOBIN_DIR):$(PATH) +PATH := $(TOOLS_GOBIN_DIR):$(PATH) -GOLANGCI_LINT_VER := v2.1.6 -GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) -GOLANGCI_LINT_FLAGS ?= +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint $(GOLANGCI_LINT): - GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/v2/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + make -C ../../../../.. $(GOLANGCI_LINT) -tools: $(CODE_GENERATOR) +tools: $(CODE_GENERATOR) $(GOLANGCI_LINT) .PHONY: tools codegen: diff --git a/staging/src/github.com/kcp-dev/code-generator/Makefile b/staging/src/github.com/kcp-dev/code-generator/Makefile index 993687697e5..1721fbbe25f 100644 --- a/staging/src/github.com/kcp-dev/code-generator/Makefile +++ b/staging/src/github.com/kcp-dev/code-generator/Makefile @@ -16,22 +16,17 @@ SHELL := /usr/bin/env bash KCP_ROOT_DIR ?= $(abspath ../../../../..) -GO_INSTALL = $(KCP_ROOT_DIR)/hack/go-install.sh - BUILD_DEST ?= _build BUILDFLAGS ?= CMD ?= $(notdir $(wildcard ./cmd/*)) -TOOLS_DIR=hack/tools -GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -TMPDIR := $(shell mktemp -d) +TOOLS_DIR = hack/tools +TOOLS_GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -GOLANGCI_LINT_VER := v2.1.6 -GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint $(GOLANGCI_LINT): - GOBIN=$(GOBIN_DIR) $(GO_INSTALL) github.com/golangci/golangci-lint/cmd/golangci-lint $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + make -C ../../../../.. $(GOLANGCI_LINT) .PHONY: imports imports: WHAT ?= From 7c189fb8defc8271e4ae63285cc7696259777e72 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Tue, 4 Nov 2025 13:39:31 +0100 Subject: [PATCH 3/9] =?UTF-8?q?update=20to=20=C2=B5get=200.2.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-behalf-of: @SAP christoph.mewes@sap.com --- hack/uget.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/uget.sh b/hack/uget.sh index f90e5739e93..ef5de24890b 100755 --- a/hack/uget.sh +++ b/hack/uget.sh @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget # SPDX-License-Identifier: MIT # -# µget 0.2.0 – your friendly downloader +# µget 0.2.2 – your friendly downloader # ------------------------------------- # # µget can download software as binaries, archives or Go modules. @@ -121,7 +121,7 @@ uget::checksum::read() { local kvString="$1" if [[ -f "$UGET_CHECKSUMS" ]]; then - awk -F'|' "{ if (\$1 == \"$BINARY\" && \$2 == \"$kvString\") print \$3 }" "$UGET_CHECKSUMS" + awk -F'|' -v "binary=$BINARY" -v "kv=$kvString" '{ if ($1 == binary && $2 == kv) print $3 }' "$UGET_CHECKSUMS" fi } @@ -141,9 +141,9 @@ uget::checksum::write() { # (for better readability, do not invert the condition here); # grep will drop any empty lines awk \ - -F'|' \ - "{ if (\$1 == \"$BINARY\" && \$2 == \"$kvString\") {} else print }" \ - "$UGET_CHECKSUMS" | grep . > "$tempDir/checksums.txt" + -F'|' -v "binary=$BINARY" -v "kv=$kvString" \ + '{ if ($1 == binary && $2 == kv) {} else print }' \ + "$UGET_CHECKSUMS" | (grep . || true) > "$tempDir/checksums.txt" # add our new checksum echo "$BINARY|$kvString|$checksum" >> "$tempDir/checksums.txt" @@ -469,7 +469,7 @@ if $UGET_UPDATE; then # download binary into tempdir, update checksums, but then delete it again uget::update "$kvString" "$url" - done < <(awk -F'|' "{ if (\$1 == \"$BINARY\") print \$2 }" "$UGET_CHECKSUMS") + done < <(awk -F'|' -v "binary=$BINARY" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS") uget::log "All checksums were updated." else From 2c26c7a14fbcc31bea908159606ba3349f3559c3 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 5 Nov 2025 12:38:26 +0100 Subject: [PATCH 4/9] use binaries for controller-gen On-behalf-of: @SAP christoph.mewes@sap.com --- Makefile | 2 +- hack/tools.checksums | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b0500ce9ed1..9b4b1d5fee4 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ tools: $(GOLANGCI_LINT) $(HTTEST) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PAT .PHONY: tools $(CONTROLLER_GEN): - @GO_MODULE=true hack/uget.sh sigs.k8s.io/controller-tools/cmd/controller-gen ${CONTROLLER_GEN_BIN} $(CONTROLLER_GEN_VER) + @UNCOMPRESSED=true hack/uget.sh https://github.com/kubernetes-sigs/controller-tools/releases/download/{VERSION}/controller-gen-{GOOS}-{GOARCH} ${CONTROLLER_GEN_BIN} $(CONTROLLER_GEN_VER) controller-gen* $(YAML_PATCH): @GO_MODULE=true hack/uget.sh github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) diff --git a/hack/tools.checksums b/hack/tools.checksums index 108f1f5e02c..bc447c4f91f 100644 --- a/hack/tools.checksums +++ b/hack/tools.checksums @@ -1,6 +1,6 @@ -controller-gen|GOARCH=amd64;GOOS=linux|cc3588262701701c3676fdf11094ccb5633650a2fc2cab9b237a0ea83214a75e -controller-gen|GOARCH=arm64;GOOS=darwin|e44b122bfc0911d98db7d5f6129c08dfd3df4da1dc8692c4aef4a8bd903edfcf -controller-gen|GOARCH=arm64;GOOS=linux|3514501d47db429950feaea34836ed59813c1f8421fbbbb8217884eb125c0cf7 +controller-gen|GOARCH=amd64;GOOS=linux|21e5f3239666fc0c5e2d23c2a3a83fd655af40a969ede7a118b86832c35a829f +controller-gen|GOARCH=arm64;GOOS=darwin|2ca28be7185d9279ed82e3355529b0543938f392cb812add3f25a62196ed7441 +controller-gen|GOARCH=arm64;GOOS=linux|a1a1f758435d05933c4b2f8c292f8ab2448e81a02c45f14dbd81c10e87ec4b20 golangci-lint|GOARCH=amd64;GOOS=linux|7009324a8aad93c1f84dce1c3cf61181bdd6b68e5f1b8b5d6971662258255050 golangci-lint|GOARCH=arm64;GOOS=darwin|96dc1dbbb686fc7982b9ea4b800f3d234a2d9fb78b0344f287e8d28ee11bc72b golangci-lint|GOARCH=arm64;GOOS=linux|c51ff5b21be688b043baea44de7dd855cf07b855c14f0de405bfaf922b1d7634 From 6dfddb14d7cdafa4988a4355816be25488aaee31 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 6 Nov 2025 16:26:21 +0100 Subject: [PATCH 5/9] =?UTF-8?q?upgrade=20=C2=B5get=20to=200.3,=20adding=20?= =?UTF-8?q?posix=20compatibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-behalf-of: @SAP christoph.mewes@sap.com --- hack/uget.sh | 399 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 235 insertions(+), 164 deletions(-) diff --git a/hack/uget.sh b/hack/uget.sh index ef5de24890b..8dd96b001eb 100755 --- a/hack/uget.sh +++ b/hack/uget.sh @@ -1,23 +1,33 @@ -#!/usr/bin/env bash +#!/bin/sh # SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget # SPDX-License-Identifier: MIT # -# µget 0.2.2 – your friendly downloader +# µget 0.3.0 – your friendly downloader # ------------------------------------- # # µget can download software as binaries, archives or Go modules. # -# Usage: ./uget.sh URL_PATTERN VERSION -# -# µget supports a large range of environment variables to customize its -# behaviour. +# Usage: ./uget.sh URL_PATTERN BINARY_NAME VERSION [EXTRACT_PATTERN=**/$BINARY_NAME] -set -euo pipefail +set -eu ############################################################################### # Configuration +# µget supports a large range of environment variables to customize its +# behaviour. Global configuration settings are prefixed with UGET_, settings +# usually meant for a single tool only have no prefix (like GO_MODULE). + +# GO_MODULE can be set to true to create a dummy Go module, add the given +# URL and version as a dependency and then go build the desired binary. Note +# that Go modules do not use checksums by default, see $UGET_GO_CHECKSUMS. +GO_MODULE=${GO_MODULE:-false} + +# UNCOMPRESSED can be set to true if the downloaded file is not an archive, +# but the binary itself and doesn't need decompressing. +UNCOMPRESSED=${UNCOMPRESSED:-false} + # UGET_UPDATE can be set to true when the VERSION parameter of a program has # been updated and you want µget to update all known variants (based on the # checksums file). When UGET_CHECKSUMS is not in use, this variable has no @@ -60,139 +70,161 @@ UGET_GO_CHECKSUMS=${UGET_GO_CHECKSUMS:-false} ############################################################################### # Function library -uget::mktemp() { +uget_mktemp() { + set -e # --tmpdir does not work on MacOS mktemp -d -p "$ABS_UGET_TEMPDIR" } -uget::log() { - if [[ "$UGET_PRINT_PATH" == "no" ]]; then +uget_log() { + if [ "$UGET_PRINT_PATH" = "no" ]; then echo "$@" >&2 fi } -uget::error() { +uget_error() { echo "$@" >&2 } -uget::lowercase() { +uget_lowercase() { + set -e cat | tr '[:upper:]' '[:lower:]' } -uget::checksum::enabled() { - [[ -n "$UGET_CHECKSUMS" ]] +uget_checksum_enabled() { + set -e + [ -n "$UGET_CHECKSUMS" ] } -uget::checksum::check() { +uget_checksum_check() { + set -e + local kvString="$1" local downloadedBinary="$2" - if ! uget::checksum::enabled; then return; fi + if ! uget_checksum_enabled; then return; fi local newChecksum - newChecksum="$(uget::checksum::calculate "$downloadedBinary")" + newChecksum="$(uget_checksum_calculate "$downloadedBinary")" local oldChecksum - oldChecksum="$(uget::checksum::read "$kvString")" - - if [[ -n "$oldChecksum" ]] && [[ "$oldChecksum" != "$newChecksum" ]]; then - uget::error - uget::error " *************************************************************************" - uget::error " SECURITY ERROR" - uget::error - uget::error " The downloaded file $downloadedBinary does not have the expected checksum." - uget::error - uget::error " Expected: $oldChecksum" - uget::error " Actual : $newChecksum" - uget::error - uget::error " If you are updating $BINARY, this error is expected." - uget::error " Re-run this command with the environment variable UGET_UPDATE=true to make" - uget::error " µget update the checksums for all known variants of $BINARY." - uget::error " *************************************************************************" - uget::error + oldChecksum="$(uget_checksum_read "$kvString")" + + if [ -n "$oldChecksum" ] && [ "$oldChecksum" != "$newChecksum" ]; then + uget_error + uget_error " *************************************************************************" + uget_error " SECURITY ERROR" + uget_error + uget_error " The downloaded file $downloadedBinary does not have the expected checksum." + uget_error + uget_error " Expected: $oldChecksum" + uget_error " Actual : $newChecksum" + uget_error + uget_error " If you are updating $BINARY, this error is expected." + uget_error " Re-run this command with the environment variable UGET_UPDATE=true to make" + uget_error " µget update the checksums for all known variants of $BINARY." + uget_error " *************************************************************************" + uget_error return 1 fi - uget::checksum::write "$kvString" "$newChecksum" + if [ -z "$oldChecksum" ]; then + uget_checksum_write "$kvString" "$newChecksum" + fi } -uget::checksum::read() { +uget_checksum_read() { + set -e + local kvString="$1" - if [[ -f "$UGET_CHECKSUMS" ]]; then + if [ -f "$UGET_CHECKSUMS" ]; then awk -F'|' -v "binary=$BINARY" -v "kv=$kvString" '{ if ($1 == binary && $2 == kv) print $3 }' "$UGET_CHECKSUMS" fi } -uget::checksum::calculate() { +uget_checksum_calculate() { + set -e "$UGET_HASHFUNC" "$1" | awk '{ print $1 }' } -uget::checksum::write() { +uget_checksum_write() { + set -e + local kvString="$1" local checksum="$2" - if [[ -f "$UGET_CHECKSUMS" ]]; then + if [ -f "$UGET_CHECKSUMS" ]; then local tempDir - tempDir="$(uget::mktemp)" + tempDir="$(uget_mktemp)" # use awk to drop any existing hash for this binary/keyvalue combo # (for better readability, do not invert the condition here); - # grep will drop any empty lines + # checking for NF (number of fields) to drop empty lines awk \ -F'|' -v "binary=$BINARY" -v "kv=$kvString" \ - '{ if ($1 == binary && $2 == kv) {} else print }' \ - "$UGET_CHECKSUMS" | (grep . || true) > "$tempDir/checksums.txt" + '{ if (NF == 0 || ($1 == binary && $2 == kv)) {} else print }' \ + "$UGET_CHECKSUMS" > "$tempDir/checksums.txt" # add our new checksum echo "$BINARY|$kvString|$checksum" >> "$tempDir/checksums.txt" # sort the file because it looks nicer and prevents ugly git diffs - cat "$tempDir/checksums.txt" | sort > "$UGET_CHECKSUMS" + sort "$tempDir/checksums.txt" > "$UGET_CHECKSUMS" rm -rf -- "$tempDir" else # start a new file - echo "$BINARY|$kvString|$checksum" >> "$UGET_CHECKSUMS" + echo "$BINARY|$kvString|$checksum" > "$UGET_CHECKSUMS" fi } -uget::url::placeholder() { +uget_url_placeholder() { + set -e + case "$1" in GOARCH) go env GOARCH ;; GOOS) go env GOOS ;; - UARCH) uname -m | uget::lowercase ;; - UOS) uname -s | uget::lowercase ;; - *) uget::error "Unexpected placeholder $1."; return 1 ;; + UARCH) uname -m | uget_lowercase ;; + UOS) uname -s | uget_lowercase ;; + *) uget_error "Unexpected placeholder $1."; return 1 ;; esac } -# valueFromPair returns "foo" for ";myvalue=foo;myothervalue=bar;" when called with -# "myvalue" as the key. The kv string must already be surrounded with semicolons. -uget::url::valueFromPair() { +# valueFromPair returns "foo" for "myvalue=foo;myothervalue=bar" when called with +# "myvalue" as the key. +uget_url_valueFromPair() { + set -e + local kvString="$1" local key="$2" - (echo "$kvString" | grep -oE ";$key=([^;]+)" || true) | cut -f2 -d'=' + # adding semicolons makes matching full keys easier + echo ";$kvString;" | sed -E "s/.*;$key=([^;]+).*/\\1/" } -uget::url::setKeyInPairs() { +uget_url_setKeyInPairs() { + set -e + local kvString="$1" local key="$2" local value="$3" - # first take the existing pairs but without the one for the given key, + # first take the existing pairs and turn it into a multiline string; then + # awk out the existing pair for $key, # then add a new pair and sort it all together, strip empty lines in case # kvstring as empty, then join the multiple lines back into a single line # and drop the trailing ';' ( - echo "$kvString" | tr ';' "\n" | (grep -vE "^$key=" || true) + echo "$kvString" | tr ';' "\n" | awk -F'=' -v "key=$key" '{ if ($1 != key) print }' echo "$key=$value" ) | sort | sed '/^[[:space:]]*$/d' | tr "\n" ';' | sed 's/;$//' } -uget::url::findPlaceholders() { +uget_url_findPlaceholders() { + set -e + # match all {...}, # then sort and return only unique values (no need to replace the # same placeholder multiple times) (this is important to allow for consistent @@ -208,23 +240,26 @@ uget::url::findPlaceholders() { # replaceLive() use live system-data to replace placeholders in the given pattern. # It returns a string of form "KVSTRING|URL". -uget::url::replaceLive() { +uget_url_replaceLive() { + set -e + local urlPattern="$1" local usedPlaceholders="" - for placeholder in $(uget::url::findPlaceholders "$urlPattern"); do + for placeholder in $(uget_url_findPlaceholders "$urlPattern"); do # version is treated specially - [[ "$placeholder" == "VERSION" ]] && continue + [ "$placeholder" = "VERSION" ] && continue - replacement="$(uget::url::placeholder "$placeholder")" - urlPattern="${urlPattern//\{$placeholder\}/$replacement}" + local replacement + replacement="$(uget_url_placeholder "$placeholder")" + urlPattern="$(echo "$urlPattern" | sed "s|{$placeholder}|$replacement|g")" # remember this placeholder and its value usedPlaceholders="$usedPlaceholders;$placeholder=$replacement" done - # trim leading ";" (this is safe even for zero-length strings) - usedPlaceholders="${usedPlaceholders:1}" + # trim leading ";" + usedPlaceholders="$(echo "$usedPlaceholders" | sed 's/^;//')" echo "$usedPlaceholders|$urlPattern" } @@ -233,64 +268,137 @@ uget::url::replaceLive() { # a placeholder, but uses a given key-value pair string as the source. It also # only returns the resulting string, since the used placeholders are known to # the caller already. -uget::url::replaceWithArgs() { +uget_url_replaceWithArgs() { + set -e + local urlPattern="$1" local kvString="$2" - # make matching easier - kvString=";$kvString;" - - for placeholder in $(uget::url::findPlaceholders "$urlPattern"); do + for placeholder in $(uget_url_findPlaceholders "$urlPattern"); do # version is treated specially - [[ "$placeholder" == "VERSION" ]] && continue + [ "$placeholder" = "VERSION" ] && continue - replacement="$(uget::url::valueFromPair "$kvString" "$placeholder")" - if [[ -z "$replacement" ]]; then - uget::error "Found no replacement string for placeholder $placeholder." + local replacement + replacement="$(uget_url_valueFromPair "$kvString" "$placeholder")" + if [ -z "$replacement" ]; then + uget_error "Found no replacement string for placeholder $placeholder." exit 1 fi - urlPattern="${urlPattern//\{$placeholder\}/$replacement}" + urlPattern="$(echo "$urlPattern" | sed "s|{$placeholder}|$replacement|g")" done echo "$urlPattern" } # returns "KVSTRING URL" -uget::url::build() { +uget_url_build() { + set -e + local kvString="${1:-}" local pattern - if [[ -z "$kvString" ]]; then - result="$(uget::url::replaceLive "$URL_PATTERN")" + if [ -z "$kvString" ]; then + local result + result="$(uget_url_replaceLive "$URL_PATTERN")" kvString="$(echo "$result" | cut -d'|' -f1)" pattern="$(echo "$result" | cut -d'|' -f2)" else - pattern="$(uget::url::replaceWithArgs "$URL_PATTERN" "$kvString")" + pattern="$(uget_url_replaceWithArgs "$URL_PATTERN" "$kvString")" fi - pattern="${pattern//\{VERSION\}/$VERSION}" + pattern="$(echo "$pattern" | sed "s|{VERSION}|$VERSION|g")" echo "$kvString|$pattern" } -# uget::fetch() detects whether wget or curl is available for downloading something. -uget::fetch() { +# uget_http_download() uses either curl or wget to download the given URL into +# the current directory. +uget_http_download() { + set -e + local url="$1" - if command -v curl &> /dev/null; then + if command -v curl >/dev/null 2>&1; then curl --fail -LO "$url" - elif command -v wget &> /dev/null; then + elif command -v wget >/dev/null 2>&1; then wget "$url" else - uget::error "Neither curl nor wget are available." + uget_error "Neither curl nor wget are available." return 1 fi } -# uget::download performs the actual download +# uget_go_install creates a temporary Go module, adds the given URL as a +# dependency and then builds $BINARY. +uget_go_install() { + set -e + + local destinationDir="$1" + local kvString="$2" + local url="$3" + + # make sure GOARCH and GOOS will be set correctly + local os + local arch + os="$(uget_url_valueFromPair "$kvString" "GOOS")" + arch="$(uget_url_valueFromPair "$kvString" "GOARCH")" + + # since we crosscompile, we cannot do "GOBIN=(somewhere) go install url@version", + # because Go considers this to be dangerous behaviour: + # https://github.com/golang/go/issues/57485 + # Instead we create a dummy module and use the desired program as a dependency, + # *then* we're allowed to crosscompile it anywhere using "go build". Go figure. + + go mod init temp 2>/dev/null + go get "$url@$VERSION" + + # go build command is meant to be expanded + # shellcheck disable=SC2086 + GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" $UGET_GO_BUILD_CMD -o "$BINARY" "$url" + + mv "$BINARY" "$destinationDir/$BINARY" +} + +# uget_extract_archive extracts a downloaded archive and moves the one interesting +# file out of it. +uget_extract_archive() { + set -e + + local destinationDir="$1" + local archive="$2" + + if ! $UNCOMPRESSED; then + case "$archive" in + *.tar.gz | *.tgz) + tar xzf "$archive" + ;; + *.tar.bz2 | *.tbz2) + tar xjf "$archive" + ;; + *.tar.xz | *.txz) + tar xJf "$archive" + ;; + *.zip) + unzip "$archive" + ;; + *) + uget_error "Unknown file type: $archive" + return 1 + esac + fi + + # pattern is explicitly meant to be interpreted by the shell + # shellcheck disable=SC2086 + mv $BINARY_PATTERN "$destinationDir/$BINARY" + chmod +x "$destinationDir/$BINARY" +} + +# uget_download performs the actual download # and places the binary in a given directory. -uget::download() { +uget_download() { + set -e + local destinationDir="$1" local kvString="$2" local url="$3" @@ -299,55 +407,19 @@ uget::download() { startDir="$(pwd)" local tempDir - tempDir="$(uget::mktemp)" + tempDir="$(uget_mktemp)" cd "$tempDir" if $GO_MODULE; then - # make sure GOARCH and GOOS are set correctly - os="$(uget::url::valueFromPair ";$kvString;" "GOOS")" - arch="$(uget::url::valueFromPair ";$kvString;" "GOARCH")" - - # since we crosscompile, we cannot do "GOBIN=(somewhere) go install url@version", - # because Go considers this to be dangerous behaviour: - # https://github.com/golang/go/issues/57485 - # Instead we create a dummy module and use the desired program as a dependency, - # *then* we're allowed to crosscompile it anywhere using "go build". Go figure. - - go mod init temp 2>/dev/null - go get "$url@$VERSION" - - GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" GOBIN=$(realpath .) $UGET_GO_BUILD_CMD -o "$BINARY" "$url" - - mv "$BINARY" "$destinationDir/$BINARY" + uget_go_install "$destinationDir" "$kvString" "$url" else - uget::fetch "$url" - archive="$(ls)" + uget_http_download "$url" - if ! $UNCOMPRESSED; then - case "$archive" in - *.tar.gz | *.tgz) - tar xzf "$archive" - ;; - *.tar.bz2 | *.tbz2) - tar xjf "$archive" - ;; - *.tar.xz | *.txz) - tar xJf "$archive" - ;; - *.zip) - unzip "$archive" - ;; - *) - uget::error "Unknown file type: $archive" - return 1 - esac - fi + local archive + archive="$(ls)" - # pattern is explicitly meant to be interpreted by the shell - # shellcheck disable=SC2086 - mv $BINARY_PATTERN "$destinationDir/$BINARY" - chmod +x "$destinationDir/$BINARY" + uget_extract_archive "$destinationDir" "$archive" fi cd "$startDir" @@ -355,15 +427,17 @@ uget::download() { } # ready() checks if the desired binary already exists in the desired version. -uget::ready() { +uget_ready() { local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" local versionFile="$fullFinalPath.version" - [ -f "$fullFinalPath" ] && [ -f "$versionFile" ] && [ "$VERSION" == "$(cat "$versionFile")" ] + [ -f "$fullFinalPath" ] && [ -f "$versionFile" ] && [ "$VERSION" = "$(cat "$versionFile")" ] } # install() downloads the binary, checks the checksum and places it in UGET_DIRECTORY. -uget::install() { +uget_install() { + set -e + local kvString="$1" local url="$2" local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" @@ -373,26 +447,28 @@ uget::install() { startDir="$(pwd)" local tempDir - tempDir="$(uget::mktemp)" + tempDir="$(uget_mktemp)" - uget::log "Downloading $BINARY version $VERSION ..." - uget::download "$tempDir" "$kvString" "$url" + uget_log "Downloading $BINARY version $VERSION ..." + uget_download "$tempDir" "$kvString" "$url" - fullBinaryPath="$tempDir/$BINARY" - uget::checksum::check "$kvString" "$fullBinaryPath" + local fullBinaryPath="$tempDir/$BINARY" + uget_checksum_check "$kvString" "$fullBinaryPath" # if everything is fine, place the binary in its final location mv "$fullBinaryPath" "$fullFinalPath" - echo -n "$VERSION" > "$versionFile" + echo "$VERSION" > "$versionFile" - uget::log "Installed at $UGET_DIRECTORY/$BINARY." + uget_log "Installed at $UGET_DIRECTORY/$BINARY." cd "$startDir" rm -rf -- "$tempDir" } # update() downloads the binary, updates the checksums and discards the binary. -uget::update() { +uget_update() { + set -e + local kvString="$1" local url="$2" @@ -400,17 +476,17 @@ uget::update() { startDir="$(pwd)" local tempDir - tempDir="$(uget::mktemp)" + tempDir="$(uget_mktemp)" - uget::log " ~> $kvString" - uget::download "$tempDir" "$kvString" "$url" + uget_log " ~> $kvString" + uget_download "$tempDir" "$kvString" "$url" - fullBinaryPath="$tempDir/$BINARY" + local fullBinaryPath="$tempDir/$BINARY" local checksum - checksum="$(uget::checksum::calculate "$fullBinaryPath")" + checksum="$(uget_checksum_calculate "$fullBinaryPath")" - uget::checksum::write "$kvString" "$checksum" + uget_checksum_write "$kvString" "$checksum" cd "$startDir" rm -rf -- "$tempDir" @@ -425,11 +501,6 @@ export BINARY="$2" export VERSION="$3" BINARY_PATTERN="${4:-**/$BINARY}" -# additional per-tool configuration, which is not prefixed with UGET_ because -# it's scoped to single tools only -GO_MODULE=${GO_MODULE:-false} -UNCOMPRESSED=${UNCOMPRESSED:-false} - # ensure target directory exists mkdir -p "$UGET_DIRECTORY" "$UGET_TEMPDIR" @@ -440,12 +511,12 @@ if $GO_MODULE && ! $UGET_GO_CHECKSUMS; then UGET_CHECKSUMS="" fi -if uget::checksum::enabled; then +if uget_checksum_enabled; then touch "$UGET_CHECKSUMS" UGET_CHECKSUMS="$(realpath "$UGET_CHECKSUMS")" else if $UGET_UPDATE && $GO_MODULE; then - uget::error "Checksums are disabled for Go modules, cannot update them." + uget_error "Checksums are disabled for Go modules, cannot update them." exit 1 fi @@ -460,22 +531,22 @@ fi # recalculate the checksums and then discard the temporary binaries. if $UGET_UPDATE; then - uget::log "Updating checksums for $BINARY ..." + uget_log "Updating checksums for $BINARY ..." # Find and process all known variants... - while read -r kvString; do - result="$(uget::url::build "$kvString")" + awk -F'|' -v "binary=$BINARY" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS" | while IFS= read -r kvString; do + result="$(uget_url_build "$kvString")" url="$(echo "$result" | cut -d'|' -f2)" # download binary into tempdir, update checksums, but then delete it again - uget::update "$kvString" "$url" - done < <(awk -F'|' -v "binary=$BINARY" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS") + uget_update "$kvString" "$url" + done - uget::log "All checksums were updated." + uget_log "All checksums were updated." else - if ! uget::ready; then + if ! uget_ready; then # Replace placeholders in the URL with system-specific data, like arch or OS - result="$(uget::url::build)" + result="$(uget_url_build)" kvString="$(echo "$result" | cut -d'|' -f1)" url="$(echo "$result" | cut -d'|' -f2)" @@ -483,11 +554,11 @@ else # the resulting binary still depends on the local system. To support checksums # for different machines, we always assume GOARCH and GOOS are "used placeholders". if $GO_MODULE; then - kvString="$(uget::url::setKeyInPairs "$kvString" "GOARCH" "$(go env GOARCH)")" - kvString="$(uget::url::setKeyInPairs "$kvString" "GOOS" "$(go env GOOS)")" + kvString="$(uget_url_setKeyInPairs "$kvString" "GOARCH" "$(uget_url_placeholder GOARCH)")" + kvString="$(uget_url_setKeyInPairs "$kvString" "GOOS" "$(uget_url_placeholder GOOS)")" fi - uget::install "$kvString" "$url" + uget_install "$kvString" "$url" fi case "$UGET_PRINT_PATH" in From cee78433cf9257ef2be697dfc920b0b9a94a4d0f Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 6 Nov 2025 16:32:44 +0100 Subject: [PATCH 6/9] =?UTF-8?q?mark=20these=20targets=20as=20PHONY=20becau?= =?UTF-8?q?se=20=C2=B5get=20cannot=20(yet=3F)=20handle=20binaries=20with?= =?UTF-8?q?=20versions=20appended=20to=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-behalf-of: @SAP christoph.mewes@sap.com --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 9b4b1d5fee4..31a9379d2a5 100644 --- a/Makefile +++ b/Makefile @@ -128,24 +128,28 @@ install: require-jq require-go require-git verify-go-versions ## Install the pro done .PHONY: install +.PHONY: $(GOLANGCI_LINT) $(GOLANGCI_LINT): @hack/uget.sh \ https://github.com/golangci/golangci-lint/releases/download/v{VERSION}/golangci-lint-{VERSION}-{GOOS}-{GOARCH}.tar.gz \ ${GOLANGCI_LINT_BIN} \ ${GOLANGCI_LINT_VER} +.PHONY: $(HTTEST) $(HTTEST): @hack/uget.sh \ https://codeberg.org/xrstf/httest/releases/download/v{VERSION}/httest_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ ${HTTEST_BIN} \ ${HTTEST_VER} +.PHONY: $(LOGCHECK) $(LOGCHECK): @GO_MODULE=true hack/uget.sh \ sigs.k8s.io/logtools/logcheck \ ${LOGCHECK_BIN} \ $(LOGCHECK_VER) +.PHONY: $(KCP_APIGEN_GEN) $(KCP_APIGEN_GEN): pushd . && cd staging/src/github.com/kcp-dev/sdk && GOBIN=$(TOOLS_GOBIN_DIR) go install ./cmd/apigen && popd @@ -205,12 +209,15 @@ vendor: ## Vendor the dependencies tools: $(GOLANGCI_LINT) $(HTTEST) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PATCH) $(GOTESTSUM) ## Install tools .PHONY: tools +.PHONY: $(CONTROLLER_GEN) $(CONTROLLER_GEN): @UNCOMPRESSED=true hack/uget.sh https://github.com/kubernetes-sigs/controller-tools/releases/download/{VERSION}/controller-gen-{GOOS}-{GOARCH} ${CONTROLLER_GEN_BIN} $(CONTROLLER_GEN_VER) controller-gen* +.PHONY: $(YAML_PATCH) $(YAML_PATCH): @GO_MODULE=true hack/uget.sh github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) +.PHONY: $(GOTESTSUM) $(GOTESTSUM): @hack/uget.sh \ https://github.com/gotestyourself/gotestsum/releases/download/v{VERSION}/gotestsum_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ From 5458bf8182c4f3e893b30835b4f893d0be5aa2bb Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 6 Nov 2025 18:33:50 +0100 Subject: [PATCH 7/9] =?UTF-8?q?update=20=C2=B5get=20to=200.3.2=20to=20make?= =?UTF-8?q?=20usage=20more=20comfortable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-behalf-of: @SAP christoph.mewes@sap.com --- hack/uget.sh | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/hack/uget.sh b/hack/uget.sh index 8dd96b001eb..d10dcd50384 100755 --- a/hack/uget.sh +++ b/hack/uget.sh @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget # SPDX-License-Identifier: MIT # -# µget 0.3.0 – your friendly downloader +# µget 0.3.2 – your friendly downloader # ------------------------------------- # # µget can download software as binaries, archives or Go modules. @@ -30,10 +30,16 @@ UNCOMPRESSED=${UNCOMPRESSED:-false} # UGET_UPDATE can be set to true when the VERSION parameter of a program has # been updated and you want µget to update all known variants (based on the -# checksums file). When UGET_CHECKSUMS is not in use, this variable has no -# effect. +# checksums file) before installing the correct variant for the current system. +# When UGET_CHECKSUMS is not in use, this variable has no effect. +# Use UGET_UPDATE_ONLY if you want to just update the checksums without also +# installing the binary. UGET_UPDATE=${UGET_UPDATE:-false} +# UGET_UPDATE_ONLY is like UGET_UPDATE, but does not install the binary on +# the current system (i.e. it only touches the checksum file). +UGET_UPDATE_ONLY=${UGET_UPDATE_ONLY:-false} + # UGET_DIRECTORY is where downloaded binaries will be placed. UGET_DIRECTORY="${UGET_DIRECTORY:-_tools}" @@ -123,6 +129,8 @@ uget_checksum_check() { uget_error " If you are updating $BINARY, this error is expected." uget_error " Re-run this command with the environment variable UGET_UPDATE=true to make" uget_error " µget update the checksums for all known variants of $BINARY." + uget_error " Use UGET_UPDATE_ONLY=true if you want to just update the checksums and not" + uget_error " install the given binary on this machine." uget_error " *************************************************************************" uget_error @@ -515,22 +523,30 @@ if uget_checksum_enabled; then touch "$UGET_CHECKSUMS" UGET_CHECKSUMS="$(realpath "$UGET_CHECKSUMS")" else - if $UGET_UPDATE && $GO_MODULE; then - uget_error "Checksums are disabled for Go modules, cannot update them." - exit 1 + if $GO_MODULE; then + if $UGET_UPDATE || $UGET_UPDATE_ONLY; then + uget_error "Checksums are disabled for Go modules, cannot update $BINARY checksums." + # This is not an error because in complex Makefiles, there might be 3 binaries required + # for one make target, and if one of them is a Go module and you have no direct way + # to just update a single binary, then running "UGET_UPDATE make complex-task" would + # fail at the Go module. It's simply more convenient to just warn and move on to the next + # binary. + fi fi UGET_UPDATE=false + UGET_UPDATE_ONLY=false fi ############################################################################### # Main application logic -# When in update mode, we do not download the binary for the current system +# When in update-only mode, we do not download the binary for the current system # only, but instead for all known variants based on the checksums file, -# recalculate the checksums and then discard the temporary binaries. +# recalculate the checksums and then discard the temporary binaries. Otherwise +# after updating the checksums we continue with the regular install logic. -if $UGET_UPDATE; then +if $UGET_UPDATE || $UGET_UPDATE_ONLY; then uget_log "Updating checksums for $BINARY ..." # Find and process all known variants... @@ -543,7 +559,9 @@ if $UGET_UPDATE; then done uget_log "All checksums were updated." -else +fi + +if ! $UGET_UPDATE_ONLY; then if ! uget_ready; then # Replace placeholders in the URL with system-specific data, like arch or OS result="$(uget_url_build)" From b9fd216061ad860d48db2e4ab0091c658989b124 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 6 Nov 2025 18:52:42 +0100 Subject: [PATCH 8/9] =?UTF-8?q?upgrade=20to=20=C2=B5get=200.3.3,=20which?= =?UTF-8?q?=20can=20handle=20binaries=20with=20versions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-behalf-of: @SAP christoph.mewes@sap.com --- Makefile | 24 +++---- hack/uget.sh | 67 +++++++++++++------ .../src/github.com/kcp-dev/client-go/Makefile | 8 ++- .../kcp-dev/code-generator/Makefile | 8 ++- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 31a9379d2a5..013fde9131a 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ endif TOOLS_DIR = hack/tools export UGET_DIRECTORY = $(TOOLS_DIR) export UGET_CHECKSUMS = hack/tools.checksums +export UGET_VERSIONED_BINARIES = true ROOT_DIR=$(abspath .) TOOLS_GOBIN_DIR := $(abspath $(TOOLS_DIR)) GOBIN_DIR=$(abspath ./bin) @@ -42,30 +43,30 @@ endif CONTROLLER_GEN_VER := v0.17.3 CONTROLLER_GEN_BIN := controller-gen -CONTROLLER_GEN := $(TOOLS_DIR)/$(CONTROLLER_GEN_BIN) +CONTROLLER_GEN := $(TOOLS_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER) export CONTROLLER_GEN # so hack scripts can use it YAML_PATCH_VER ?= v0.0.11 YAML_PATCH_BIN := yaml-patch -YAML_PATCH := $(TOOLS_DIR)/$(YAML_PATCH_BIN) +YAML_PATCH := $(TOOLS_DIR)/$(YAML_PATCH_BIN)-$(YAML_PATCH_VER) export YAML_PATCH # so hack scripts can use it GOLANGCI_LINT_VER := 2.1.6 GOLANGCI_LINT_BIN := golangci-lint -GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER) GOLANGCI_LINT_FLAGS ?= HTTEST_VER := 0.3.4 HTTEST_BIN := httest -HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN) +HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) GOTESTSUM_VER := 1.12.3 GOTESTSUM_BIN := gotestsum -GOTESTSUM := $(abspath $(TOOLS_DIR))/$(GOTESTSUM_BIN) +GOTESTSUM := $(abspath $(TOOLS_DIR))/$(GOTESTSUM_BIN)-$(GOTESTSUM_VER) LOGCHECK_VER := d35c84c015fe03a1421e5f2ce1e3c0c3bc38d077 LOGCHECK_BIN := logcheck -LOGCHECK := $(TOOLS_GOBIN_DIR)/$(LOGCHECK_BIN) +LOGCHECK := $(TOOLS_GOBIN_DIR)/$(LOGCHECK_BIN)-$(LOGCHECK_VER) export LOGCHECK # so hack scripts can use it KCP_APIGEN_BIN := apigen @@ -128,21 +129,23 @@ install: require-jq require-go require-git verify-go-versions ## Install the pro done .PHONY: install -.PHONY: $(GOLANGCI_LINT) $(GOLANGCI_LINT): @hack/uget.sh \ https://github.com/golangci/golangci-lint/releases/download/v{VERSION}/golangci-lint-{VERSION}-{GOOS}-{GOARCH}.tar.gz \ ${GOLANGCI_LINT_BIN} \ ${GOLANGCI_LINT_VER} -.PHONY: $(HTTEST) +# help staging modules to install the same linter version +golangci-lint-version: + @echo $(GOLANGCI_LINT_VER) +.PHONY: golangci-lint-version + $(HTTEST): @hack/uget.sh \ https://codeberg.org/xrstf/httest/releases/download/v{VERSION}/httest_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ ${HTTEST_BIN} \ ${HTTEST_VER} -.PHONY: $(LOGCHECK) $(LOGCHECK): @GO_MODULE=true hack/uget.sh \ sigs.k8s.io/logtools/logcheck \ @@ -209,15 +212,12 @@ vendor: ## Vendor the dependencies tools: $(GOLANGCI_LINT) $(HTTEST) $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $(YAML_PATCH) $(GOTESTSUM) ## Install tools .PHONY: tools -.PHONY: $(CONTROLLER_GEN) $(CONTROLLER_GEN): @UNCOMPRESSED=true hack/uget.sh https://github.com/kubernetes-sigs/controller-tools/releases/download/{VERSION}/controller-gen-{GOOS}-{GOARCH} ${CONTROLLER_GEN_BIN} $(CONTROLLER_GEN_VER) controller-gen* -.PHONY: $(YAML_PATCH) $(YAML_PATCH): @GO_MODULE=true hack/uget.sh github.com/pivotal-cf/yaml-patch/cmd/yaml-patch $(YAML_PATCH_BIN) $(YAML_PATCH_VER) -.PHONY: $(GOTESTSUM) $(GOTESTSUM): @hack/uget.sh \ https://github.com/gotestyourself/gotestsum/releases/download/v{VERSION}/gotestsum_{VERSION}_{GOOS}_{GOARCH}.tar.gz \ diff --git a/hack/uget.sh b/hack/uget.sh index d10dcd50384..c59c0a4000c 100755 --- a/hack/uget.sh +++ b/hack/uget.sh @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: 2025 Christoph Mewes, https://codeberg.org/xrstf/uget # SPDX-License-Identifier: MIT # -# µget 0.3.2 – your friendly downloader +# µget 0.3.3 – your friendly downloader # ------------------------------------- # # µget can download software as binaries, archives or Go modules. @@ -48,6 +48,13 @@ UGET_DIRECTORY="${UGET_DIRECTORY:-_tools}" # tampering on the server side. UGET_CHECKSUMS="${UGET_CHECKSUMS:-}" +# UGET_VERSIONED_BINARIES can be set to true to append the binary's version to +# its filename, leading to final paths like "_tools/mytool-v1.2.3". This can +# help when µget is being used in Makefiles to improve staleness detection. +# Note that µget never deletes any installed binaries, so enabling this can +# lead to leftover binaries that you can cleanup at your own convenience. +UGET_VERSIONED_BINARIES=${UGET_VERSIONED_BINARIES:-false} + # UGET_TEMPDIR is the root directory to use when creating new temporary dirs. UGET_TEMPDIR="${UGET_TEMPDIR:-/tmp}" @@ -126,9 +133,9 @@ uget_checksum_check() { uget_error " Expected: $oldChecksum" uget_error " Actual : $newChecksum" uget_error - uget_error " If you are updating $BINARY, this error is expected." + uget_error " If you are updating $IDENTIFIER, this error is expected." uget_error " Re-run this command with the environment variable UGET_UPDATE=true to make" - uget_error " µget update the checksums for all known variants of $BINARY." + uget_error " µget update the checksums for all known variants of $IDENTIFIER." uget_error " Use UGET_UPDATE_ONLY=true if you want to just update the checksums and not" uget_error " install the given binary on this machine." uget_error " *************************************************************************" @@ -148,7 +155,7 @@ uget_checksum_read() { local kvString="$1" if [ -f "$UGET_CHECKSUMS" ]; then - awk -F'|' -v "binary=$BINARY" -v "kv=$kvString" '{ if ($1 == binary && $2 == kv) print $3 }' "$UGET_CHECKSUMS" + awk -F'|' -v "binary=$IDENTIFIER" -v "kv=$kvString" '{ if ($1 == binary && $2 == kv) print $3 }' "$UGET_CHECKSUMS" fi } @@ -171,12 +178,12 @@ uget_checksum_write() { # (for better readability, do not invert the condition here); # checking for NF (number of fields) to drop empty lines awk \ - -F'|' -v "binary=$BINARY" -v "kv=$kvString" \ + -F'|' -v "binary=$IDENTIFIER" -v "kv=$kvString" \ '{ if (NF == 0 || ($1 == binary && $2 == kv)) {} else print }' \ "$UGET_CHECKSUMS" > "$tempDir/checksums.txt" # add our new checksum - echo "$BINARY|$kvString|$checksum" >> "$tempDir/checksums.txt" + echo "$IDENTIFIER|$kvString|$checksum" >> "$tempDir/checksums.txt" # sort the file because it looks nicer and prevents ugly git diffs sort "$tempDir/checksums.txt" > "$UGET_CHECKSUMS" @@ -184,7 +191,7 @@ uget_checksum_write() { rm -rf -- "$tempDir" else # start a new file - echo "$BINARY|$kvString|$checksum" > "$UGET_CHECKSUMS" + echo "$IDENTIFIER|$kvString|$checksum" > "$UGET_CHECKSUMS" fi } @@ -361,11 +368,13 @@ uget_go_install() { go mod init temp 2>/dev/null go get "$url@$VERSION" + local tmpFilename="__tmp.bin" + # go build command is meant to be expanded # shellcheck disable=SC2086 - GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" $UGET_GO_BUILD_CMD -o "$BINARY" "$url" + GOFLAGS=-trimpath GOARCH="$arch" GOOS="$os" $UGET_GO_BUILD_CMD -o "$tmpFilename" "$url" - mv "$BINARY" "$destinationDir/$BINARY" + mv "$tmpFilename" "$destinationDir/$BINARY" } # uget_extract_archive extracts a downloaded archive and moves the one interesting @@ -437,9 +446,17 @@ uget_download() { # ready() checks if the desired binary already exists in the desired version. uget_ready() { local fullFinalPath="$ABS_UGET_DIRECTORY/$BINARY" - local versionFile="$fullFinalPath.version" + if ! [ -f "$fullFinalPath" ]; then + return 1 + fi + + # skip further checks if we're using versioned binaries + if $UGET_VERSIONED_BINARIES; then + return 0 + fi - [ -f "$fullFinalPath" ] && [ -f "$versionFile" ] && [ "$VERSION" = "$(cat "$versionFile")" ] + local versionFile="$fullFinalPath.version" + [ -f "$versionFile" ] && [ "$VERSION" = "$(cat "$versionFile")" ] } # install() downloads the binary, checks the checksum and places it in UGET_DIRECTORY. @@ -457,15 +474,18 @@ uget_install() { local tempDir tempDir="$(uget_mktemp)" - uget_log "Downloading $BINARY version $VERSION ..." + uget_log "Downloading $IDENTIFIER version $VERSION ..." uget_download "$tempDir" "$kvString" "$url" - local fullBinaryPath="$tempDir/$BINARY" - uget_checksum_check "$kvString" "$fullBinaryPath" + local fullTempBinary="$tempDir/$BINARY" + uget_checksum_check "$kvString" "$fullTempBinary" # if everything is fine, place the binary in its final location - mv "$fullBinaryPath" "$fullFinalPath" - echo "$VERSION" > "$versionFile" + mv "$fullTempBinary" "$fullFinalPath" + + if ! $UGET_VERSIONED_BINARIES; then + echo "$VERSION" > "$versionFile" + fi uget_log "Installed at $UGET_DIRECTORY/$BINARY." @@ -505,9 +525,14 @@ uget_update() { # get CLI flags export URL_PATTERN="$1" -export BINARY="$2" +export IDENTIFIER="$2" export VERSION="$3" -BINARY_PATTERN="${4:-**/$BINARY}" +BINARY_PATTERN="${4:-**/$IDENTIFIER}" + +export BINARY="$IDENTIFIER" +if $UGET_VERSIONED_BINARIES; then + BINARY="$IDENTIFIER-$VERSION" +fi # ensure target directory exists mkdir -p "$UGET_DIRECTORY" "$UGET_TEMPDIR" @@ -525,7 +550,7 @@ if uget_checksum_enabled; then else if $GO_MODULE; then if $UGET_UPDATE || $UGET_UPDATE_ONLY; then - uget_error "Checksums are disabled for Go modules, cannot update $BINARY checksums." + uget_error "Checksums are disabled for Go modules, cannot update $IDENTIFIER checksums." # This is not an error because in complex Makefiles, there might be 3 binaries required # for one make target, and if one of them is a Go module and you have no direct way # to just update a single binary, then running "UGET_UPDATE make complex-task" would @@ -547,10 +572,10 @@ fi # after updating the checksums we continue with the regular install logic. if $UGET_UPDATE || $UGET_UPDATE_ONLY; then - uget_log "Updating checksums for $BINARY ..." + uget_log "Updating checksums for $IDENTIFIER ..." # Find and process all known variants... - awk -F'|' -v "binary=$BINARY" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS" | while IFS= read -r kvString; do + awk -F'|' -v "binary=$IDENTIFIER" '{ if ($1 == binary) print $2 }' "$UGET_CHECKSUMS" | while IFS= read -r kvString; do result="$(uget_url_build "$kvString")" url="$(echo "$result" | cut -d'|' -f2)" diff --git a/staging/src/github.com/kcp-dev/client-go/Makefile b/staging/src/github.com/kcp-dev/client-go/Makefile index 9ad80ea2da7..50546b248da 100644 --- a/staging/src/github.com/kcp-dev/client-go/Makefile +++ b/staging/src/github.com/kcp-dev/client-go/Makefile @@ -15,16 +15,18 @@ # We need bash for some conditional logic below. SHELL := /usr/bin/env bash -e -KCP_ROOT_DIR ?= $(abspath ../../../../..) +KCP_ROOT_PATH = ../../../../.. +KCP_ROOT_DIR ?= $(abspath $(KCP_ROOT_PATH)) TOOLS_DIR = hack/tools TOOLS_GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) PATH := $(TOOLS_GOBIN_DIR):$(PATH) -GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint +GOLANGCI_LINT_VER = $(shell $(MAKE) --no-print-directory -C $(KCP_ROOT_PATH) golangci-lint-version) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VER) $(GOLANGCI_LINT): - make -C ../../../../.. $(GOLANGCI_LINT) + $(MAKE) -C $(KCP_ROOT_PATH) $(GOLANGCI_LINT) tools: $(CODE_GENERATOR) $(GOLANGCI_LINT) .PHONY: tools diff --git a/staging/src/github.com/kcp-dev/code-generator/Makefile b/staging/src/github.com/kcp-dev/code-generator/Makefile index 1721fbbe25f..caf043f43c4 100644 --- a/staging/src/github.com/kcp-dev/code-generator/Makefile +++ b/staging/src/github.com/kcp-dev/code-generator/Makefile @@ -14,7 +14,8 @@ SHELL := /usr/bin/env bash -KCP_ROOT_DIR ?= $(abspath ../../../../..) +KCP_ROOT_PATH = ../../../../.. +KCP_ROOT_DIR ?= $(abspath $(KCP_ROOT_PATH)) BUILD_DEST ?= _build BUILDFLAGS ?= @@ -23,10 +24,11 @@ CMD ?= $(notdir $(wildcard ./cmd/*)) TOOLS_DIR = hack/tools TOOLS_GOBIN_DIR := $(KCP_ROOT_DIR)/$(TOOLS_DIR) -GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint +GOLANGCI_LINT_VER = $(shell $(MAKE) --no-print-directory -C $(KCP_ROOT_PATH) golangci-lint-version) +GOLANGCI_LINT := $(TOOLS_GOBIN_DIR)/golangci-lint-$(GOLANGCI_LINT_VER) $(GOLANGCI_LINT): - make -C ../../../../.. $(GOLANGCI_LINT) + $(MAKE) -C $(KCP_ROOT_PATH) $(GOLANGCI_LINT) .PHONY: imports imports: WHAT ?= From 55f22c0d376d72003fd3c9e637a0beeb8da87208 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 6 Nov 2025 19:31:25 +0100 Subject: [PATCH 9/9] fix tests not finding httest anymore On-behalf-of: @SAP christoph.mewes@sap.com --- Makefile | 2 +- test/e2e/authorizer/utils.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 013fde9131a..4d96832cc1b 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ GOLANGCI_LINT_FLAGS ?= HTTEST_VER := 0.3.4 HTTEST_BIN := httest -HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) +export HTTEST := $(TOOLS_GOBIN_DIR)/$(HTTEST_BIN)-$(HTTEST_VER) GOTESTSUM_VER := 1.12.3 GOTESTSUM_BIN := gotestsum diff --git a/test/e2e/authorizer/utils.go b/test/e2e/authorizer/utils.go index 8af1c531f93..e82b8dca56e 100644 --- a/test/e2e/authorizer/utils.go +++ b/test/e2e/authorizer/utils.go @@ -44,7 +44,13 @@ func RunWebhook(ctx context.Context, t *testing.T, port string, response string) "--listen", address, } - cmd := exec.CommandContext(ctx, "httest", args...) + // get the binary name from our Makefile + httestBinary := os.Getenv("HTTEST") + if httestBinary == "" { + httestBinary = "httest" + } + + cmd := exec.CommandContext(ctx, httestBinary, args...) if err := cmd.Start(); err != nil { cancel() t.Fatalf("Failed to start webhook: %v", err)