Merge pull request #21813 from crasbe/pr/cryptocell_use_dlcache

pkg/{c25519,driver_cryptocell_310}: use dlcache for fetching zip archives, update `pkg/Makefile.http` example

Author: crasbe

Makefile.include

@@ -352,7 +352,7 @@ ifeq (,$(and $(DOWNLOAD_TO_STDOUT),$(DOWNLOAD_TO_FILE)))
     DOWNLOAD_TO_STDOUT ?= $(if $(CURL),$(CURL) -s,$(WGET) -q -O-)
   endif
   ifeq (,$(DOWNLOAD_TO_FILE))
-    DOWNLOAD_TO_FILE ?= $(if $(WGET),$(WGET) -nv -c -O,$(CURL) -s -o)
+    DOWNLOAD_TO_FILE ?= $(DLCACHE)
   endif
 endif
 

dist/tools/dlcache/README.md

@@ -1,11 +1,24 @@
 # Introduction
 
-This script aims to cache http(s)-downloads
+This script aims to cache http(s)-downloads of (mainly) zip archives.
 
 ## How it works
 
-- if a file with the right name and md5 exists, nothing happens
-- if a file with the right name and md5 is in cache, use that
-- if a file with the right name but wrong md5 is in cache, redownload
+- if a file with the right name and SHA512 exists, nothing happens
+- if a file with the right name and SHA512 is in cache, use that
+- if a file with the right name but wrong SHA512 is in cache, redownload
   download to cache
-- after download, check md5, then copy to target
+- after download, check SHA512, then copy to target
+- if no SHA512 checksum is given, always redownload
+
+## Usage
+
+In order to stay compatible with the `$(DOWNLOAD_TO_FILE)` command,
+the command usage was changed for Release 2025.10.
+
+If no SHA512 checksum was given, the file will *always* be downloaded,
+regardless of whether it is in the cache or not.
+
+```sh
+dist/tools/dlcache/dlcache.sh <output_folder> <URL> [SHA512 checksum]
+```

dist/tools/dlcache/dlcache.sh

@@ -13,13 +13,13 @@ if [ "$(uname)" = Darwin ]; then
         local lockfile="$1"
         shift
 
-        while ! shlock -p $$ -f $lockfile; do
+        while ! shlock -p "$$" -f "$lockfile"; do
             sleep 0.2
         done
 
-        $*
+        "$@"
 
-        rm $lockfile
+        rm "$lockfile"
     }
 else
     _locked() {
@@ -28,58 +28,57 @@ else
 
         (
         flock -w 600 9 || exit 1
-        $*
+        "$@"
         ) 9>"$lockfile"
     }
 fi
 
-if [ "$(uname)" = Darwin ]; then
-    MD5="md5 -r"
-else
-    MD5=md5sum
-fi
+# shasum is supported on Linux and Darwin
+SHA512="shasum -a 512"
 
-calcmd5() {
+calcsha512() {
     local file="$1"
-    local md5="$2"
+    local sha512="$2"
+    local file_sha512
+    file_sha512=$(${SHA512} "$file" | cut -d\  -f1)
 
-    local file_md5=$(${MD5} "$file" | cut -d\  -f1)
-
-    test "$md5" = "$file_md5"
+    test "$sha512" = "$file_sha512"
 }
 
 downloader() {
     if [ -n "$(command -v wget)" ]; then
-        wget -nv "$1" -O $2
+        wget -nv "$1" -O "$2"
     elif [ -n "$(command -v curl)" ]; then
-        curl -L $1 -o $2
+        curl -L "$1" -o "$2"
     else
         _echo "$0: neither wget nor curl available!"
         return 1
     fi
 }
 
 download() {
-    local url="$1"
-    local _md5="$2"
-    local basename_url=$(basename ${url})
-    local target="${3:-${basename_url}}"
-
-    [ -f "$target" ] && {
-        # if our target file exists, check it's md5.
-        calcmd5 "$target" "$_md5" && {
-            _echo "$0: target exists, md5 matches."
+    local url="$2"
+    local _sha512="$3"
+    local basename_url
+    basename_url="$(basename "${url}")"
+    local target="${1:-"${basename_url}"}"
+
+    if [ -f "$target" ] && [ -n "$_sha512" ]; then
+        # if our target file exists and an SHA512 sum was given, check it's SHA512
+        calcsha512 "$target" "$_sha512" && {
+            _echo "$0: target exists, SHA512 matches."
             exit 0
         }
-    }
+    fi
 
-    local filename="$(basename $url)"
+    local filename
+    filename="$(basename "$url")"
     [ -f "$DLCACHE_DIR/$filename" ] && {
-        # if the file exists in cache, check it's md5 and possibly remove it.
-        if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
+        # if the file exists in cache, check it's SHA512 and possibly remove it.
+        if calcsha512 "$DLCACHE_DIR/$filename" "$_sha512"; then
             _echo "$0: getting \"$url\" from cache"
         else
-            _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
+            _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong or no checksum, re-downloading"
             rm "$DLCACHE_DIR/$filename"
         fi
     }
@@ -93,10 +92,13 @@ download() {
         _echo "$0: done downloading \"$url\""
     }
 
-    calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
-        _echo "$0: checksum mismatch!"
-        exit 1
-    }
+    # only try to calculate the checksum if a checksum was given
+    if [ -n "$_sha512" ]; then
+        calcsha512 "$DLCACHE_DIR/$filename" "$_sha512" || {
+            _echo "$0: checksum mismatch!"
+            exit 1
+        }
+    fi
 
     if [ "$target" = "-" ]; then
         cat "$DLCACHE_DIR/$filename"
@@ -105,4 +107,4 @@ download() {
     fi
 }
 
-_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@"
+_locked "$DLCACHE_DIR/$(basename "$1").locked" download "$@"

makefiles/mcuboot.mk

@@ -8,7 +8,7 @@ SIGN_BINFILE = $(BINDIR)/signed-$(APPLICATION).bin
 MCUBOOT_KEYFILE ?= $(BINDIR)/key.pem
 MCUBOOT_BIN ?= $(BINDIR)/mcuboot.bin
 MCUBOOT_BIN_URL ?= http://download.riot-os.org/mynewt.mcuboot.bin
-MCUBOOT_BIN_MD5 ?= 0c71a0589bd3709fc2d90f07a0035ce7
+MCUBOOT_BIN_SHA512 ?=
 
 export IMAGE_HDR_SIZE ?= 512
 
@@ -37,7 +37,7 @@ mcuboot: mcuboot-create-key link
 	@$(COLOR_ECHO)
 
 $(MCUBOOT_BIN):
-	$(Q)$(DLCACHE) $(MCUBOOT_BIN_URL) $(MCUBOOT_BIN_MD5) $@
+	$(Q)$(DLCACHE) $@ $(MCUBOOT_BIN_URL) $(MCUBOOT_BIN_SHA512)
 
 .PHONY: mcuboot-flash-bootloader mcuboot-flash
 

pkg/Makefile.http

@@ -3,6 +3,7 @@ PKG_URL     = http://example.com/downloads # source url of the package e.g. a gi
 PKG_VERSION = v1.2.3                       # version of the package to use e.g. a git commit/ref
 PKG_EXT     = zip                          # extension of this package
 PKG_LICENSE = MIT                          # license of the package
+PKG_SHA512  = 5bcc4310f09ea247f5b65d63afae872fb92bb5c39029f336d94503fe04feccfcd22aa42b377b0927ab7f0a19111fd5a0a6842ecab147761c8f218f7f115e0da5 # SHA512 checksum of the package
 
 ifneq ($(RIOTBASE),)
 include $(RIOTBASE)/Makefile.base
@@ -30,7 +31,9 @@ $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PK
 
 $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT):
 	# Get PKG_VERSION of package from PKG_URL
-	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT)
+	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT) $(PKG_SHA512)
+	# Note: Since Release 2025.10, DOWNLOAD_TO_FILE uses dlcache and
+	# requires an SHA512 checksum for caching!
 
 clean::
 	# Reset package to checkout state.

pkg/c25519/Makefile

@@ -26,7 +26,7 @@ $(PKG_SOURCE_DIR)/: $(PKG_ZIPFILE)
 
 $(PKG_ZIPFILE):
 	$(QQ)mkdir -p $(PKGDIRBASE)
-	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL)
+	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL) $(PKG_SHA512)
 
 clean::
 	# Reset package to checkout state.

pkg/driver_cryptocell_310/Makefile

@@ -15,7 +15,7 @@ PKG_ZIP_URL = $(PKG_URL)/$(PKG_DIR_NAME)_$(PKG_VERSION).$(PKG_EXT)
 NRF_CC310_PATH = external/nrf_cc310
 
 ifneq ($(RIOTBASE),)
-include $(RIOTBASE)/Makefile.base
+  include $(RIOTBASE)/Makefile.base
 endif
 
 .PHONY: all clean distclean
@@ -34,7 +34,7 @@ $(PKG_SOURCE_DIR)/: $(PKG_ZIPFILE)
 
 $(PKG_ZIPFILE):
 	$(QQ)mkdir -p $(PKGDIRBASE)
-	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL)
+	$(Q)$(DOWNLOAD_TO_FILE) $@ $(PKG_ZIP_URL) $(PKG_SHA512)
 
 clean::
 	rm -rf $(PKG_SOURCE_DIR)