pkg,dist/tools: repl. DOWNLOAD_TO_FILE w/ DLCACHE, enh. dlcache.sh

The dlcache script is useful to avoid unnecessary repeated downloads
of zip archives, similar to the git-cache scripts. The command order
was changed to stay compatible with DOWNLOAD_TO_FILE and the only other
usage of $(DLCACHE) was changed accordingly.

To avoid the risk of collisions, dlcache now uses SHA512 sums instead
of MD5.
If no SHA512 checksum is given to dlcache.sh, it will just act as
$(DOWNLOAD_TO_FILE).

Author: crasbe

Makefile.include

@@ -352,7 +352,7 @@ ifeq (,$(and $(DOWNLOAD_TO_STDOUT),$(DOWNLOAD_TO_FILE)))
     DOWNLOAD_TO_STDOUT ?= $(if $(CURL),$(CURL) -s,$(WGET) -q -O-)
   endif
   ifeq (,$(DOWNLOAD_TO_FILE))
-    DOWNLOAD_TO_FILE ?= $(if $(WGET),$(WGET) -nv -c -O,$(CURL) -s -o)
+    DOWNLOAD_TO_FILE ?= $(DLCACHE)
   endif
 endif
 

dist/tools/dlcache/README.md

@@ -1,11 +1,24 @@
 # Introduction
 
-This script aims to cache http(s)-downloads
+This script aims to cache http(s)-downloads of (mainly) zip archives.
 
 ## How it works
 
-- if a file with the right name and md5 exists, nothing happens
-- if a file with the right name and md5 is in cache, use that
-- if a file with the right name but wrong md5 is in cache, redownload
+- if a file with the right name and SHA512 exists, nothing happens
+- if a file with the right name and SHA512 is in cache, use that
+- if a file with the right name but wrong SHA512 is in cache, redownload
   download to cache
-- after download, check md5, then copy to target
+- after download, check SHA512, then copy to target
+- if no SHA512 checksum is given, always redownload
+
+## Usage
+
+In order to stay compatible with the `$(DOWNLOAD_TO_FILE)` command,
+the command usage was changed for Release 2025.10.
+
+If no SHA512 checksum was given, the file will *always* be downloaded,
+regardless of whether it is in the cache or not.
+
+```sh
+dist/tools/dlcache/dlcache.sh <output_folder> <URL> [SHA512 checksum]
+```

dist/tools/dlcache/dlcache.sh

@@ -33,19 +33,16 @@ else
     }
 fi
 
-if [ "$(uname)" = Darwin ]; then
-    MD5="md5 -r"
-else
-    MD5=md5sum
-fi
+# shasum is supported on Linux and Darwin
+SHA512="shasum -a 512"
 
-calcmd5() {
+calcsha512() {
     local file="$1"
-    local md5="$2"
-    local file_md5
-    file_md5=$(${MD5} "$file" | cut -d\  -f1)
+    local sha512="$2"
+    local file_sha512
+    file_sha512=$(${SHA512} "$file" | cut -d\  -f1)
 
-    test "$md5" = "$file_md5"
+    test "$sha512" = "$file_sha512"
 }
 
 downloader() {
@@ -60,28 +57,28 @@ downloader() {
 }
 
 download() {
-    local url="$1"
-    local _md5="$2"
+    local url="$2"
+    local _sha512="$3"
     local basename_url
     basename_url="$(basename "${url}")"
-    local target="${3:-"${basename_url}"}"
+    local target="${1:-"${basename_url}"}"
 
-    [ -f "$target" ] && {
-        # if our target file exists, check it's md5.
-        calcmd5 "$target" "$_md5" && {
-            _echo "$0: target exists, md5 matches."
+    if [ -f "$target" ] && [ -n "$_sha512" ]; then
+        # if our target file exists and an SHA512 sum was given, check it's SHA512
+        calcsha512 "$target" "$_sha512" && {
+            _echo "$0: target exists, SHA512 matches."
             exit 0
         }
-    }
+    fi
 
     local filename
     filename="$(basename "$url")"
     [ -f "$DLCACHE_DIR/$filename" ] && {
-        # if the file exists in cache, check it's md5 and possibly remove it.
-        if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
+        # if the file exists in cache, check it's SHA512 and possibly remove it.
+        if calcsha512 "$DLCACHE_DIR/$filename" "$_sha512"; then
             _echo "$0: getting \"$url\" from cache"
         else
-            _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
+            _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong or no checksum, re-downloading"
             rm "$DLCACHE_DIR/$filename"
         fi
     }
@@ -95,10 +92,13 @@ download() {
         _echo "$0: done downloading \"$url\""
     }
 
-    calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
-        _echo "$0: checksum mismatch!"
-        exit 1
-    }
+    # only try to calculate the checksum if a checksum was given
+    if [ -n "$_sha512" ]; then
+        calcsha512 "$DLCACHE_DIR/$filename" "$_sha512" || {
+            _echo "$0: checksum mismatch!"
+            exit 1
+        }
+    fi
 
     if [ "$target" = "-" ]; then
         cat "$DLCACHE_DIR/$filename"

makefiles/mcuboot.mk

@@ -8,7 +8,7 @@ SIGN_BINFILE = $(BINDIR)/signed-$(APPLICATION).bin
 MCUBOOT_KEYFILE ?= $(BINDIR)/key.pem
 MCUBOOT_BIN ?= $(BINDIR)/mcuboot.bin
 MCUBOOT_BIN_URL ?= http://download.riot-os.org/mynewt.mcuboot.bin
-MCUBOOT_BIN_MD5 ?= 0c71a0589bd3709fc2d90f07a0035ce7
+MCUBOOT_BIN_SHA512 ?=
 
 export IMAGE_HDR_SIZE ?= 512
 
@@ -37,7 +37,7 @@ mcuboot: mcuboot-create-key link
 	@$(COLOR_ECHO)
 
 $(MCUBOOT_BIN):
-	$(Q)$(DLCACHE) $(MCUBOOT_BIN_URL) $(MCUBOOT_BIN_MD5) $@
+	$(Q)$(DLCACHE) $@ $(MCUBOOT_BIN_URL) $(MCUBOOT_BIN_SHA512)
 
 .PHONY: mcuboot-flash-bootloader mcuboot-flash
 

pkg/Makefile.http

@@ -3,6 +3,7 @@ PKG_URL     = http://example.com/downloads # source url of the package e.g. a gi
 PKG_VERSION = v1.2.3                       # version of the package to use e.g. a git commit/ref
 PKG_EXT     = zip                          # extension of this package
 PKG_LICENSE = MIT                          # license of the package
+PKG_SHA512  = 5bcc4310f09ea247f5b65d63afae872fb92bb5c39029f336d94503fe04feccfcd22aa42b377b0927ab7f0a19111fd5a0a6842ecab147761c8f218f7f115e0da5 # SHA512 checksum of the package
 
 ifneq ($(RIOTBASE),)
 include $(RIOTBASE)/Makefile.base
@@ -30,7 +31,9 @@ $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PK
 
 $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT):
 	# Get PKG_VERSION of package from PKG_URL
-	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT)
+	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT) $(PKG_SHA512)
+	# Note: Since Release 2025.10, DOWNLOAD_TO_FILE uses dlcache and
+	# requires an SHA512 checksum for caching!
 
 clean::
 	# Reset package to checkout state.