ccache: Add ccache-remote extension for distributed compilation caching via Redis

extensions/ccache-remote.sh

@@ -0,0 +1,251 @@
+# Extension: ccache-remote
+# Enables ccache with remote Redis storage for sharing compilation cache across build hosts
+#
+# Documentation: https://ccache.dev/howto/redis-storage.html
+# See also: https://ccache.dev/manual/4.10.html#config_remote_storage
+#
+# Usage:
+#   # With Avahi/mDNS auto-discovery:
+#   ./compile.sh ENABLE_EXTENSIONS=ccache-remote BOARD=...
+#
+#   # With explicit Redis server (no Avahi needed):
+#   ./compile.sh ENABLE_EXTENSIONS=ccache-remote CCACHE_REMOTE_STORAGE="redis://192.168.1.65:6379" BOARD=...
+#
+#   # Disable local cache, use remote only (saves local disk space):
+#   ./compile.sh ENABLE_EXTENSIONS=ccache-remote CCACHE_REMOTE_ONLY=yes BOARD=...
+#
+# Automatically sets USE_CCACHE=yes
+#
+# Supported ccache environment variables (passed through to builds):
+# See: https://ccache.dev/manual/latest.html#_configuration_options
+#   CCACHE_BASEDIR        - base directory for path normalization (enables cache sharing)
+#   CCACHE_REMOTE_STORAGE - remote storage URL (redis://...)
+#   CCACHE_REMOTE_ONLY    - use only remote storage, disable local cache
+#   CCACHE_READONLY       - read-only mode, don't update cache
+#   CCACHE_RECACHE        - don't use cached results, but update cache
+#   CCACHE_RESHARE        - rewrite cache entries to remote storage
+#   CCACHE_DISABLE        - disable ccache completely
+#   CCACHE_MAXSIZE        - maximum cache size (e.g., "10G")
+#   CCACHE_MAXFILES       - maximum number of files in cache
+#   CCACHE_NAMESPACE      - cache namespace for isolation
+#   CCACHE_SLOPPINESS     - comma-separated list of sloppiness options
+#   CCACHE_UMASK          - umask for cache files
+#   CCACHE_LOGFILE        - path to log file
+#   CCACHE_DEBUGLEVEL     - debug level (1-2)
+#   CCACHE_STATSLOG       - path to stats log file
+#   CCACHE_PCH_EXTSUM     - include PCH extension in hash
+#
+# CCACHE_REMOTE_STORAGE format (ccache 4.4+):
+#   redis://HOST[:PORT][|attribute=value...]
+#   Common attributes:
+#     connect-timeout=N   - connection timeout in milliseconds (default: 100)
+#     operation-timeout=N - operation timeout in milliseconds (default: 10000)
+#   Example: "redis://192.168.1.65:6379|connect-timeout=500"
+#
+# Avahi/mDNS auto-discovery:
+#   This extension tries to resolve 'ccache.local' hostname via mDNS.
+#   To publish this hostname on Redis server, run:
+#     avahi-publish-address -R ccache.local <SERVER_IP>
+#   Or create a systemd service (see below).
+#
+#   Server setup example:
+#     1. Install: apt install redis-server avahi-daemon avahi-utils
+#     2. Configure Redis (/etc/redis/redis.conf):
+#          bind 0.0.0.0 ::
+#          protected-mode no
+#          maxmemory 4G
+#          maxmemory-policy allkeys-lru
+#        WARNING: This configuration is INSECURE - Redis is open without authentication.
+#        Use ONLY in a fully trusted private network with no internet access.
+#        For secure setup (password, TLS, ACL), see: https://redis.io/docs/management/security/
+#     3. Publish hostname (replace IP_ADDRESS with actual IP):
+#          avahi-publish-address -R ccache.local IP_ADDRESS
+#        Or as systemd service /etc/systemd/system/ccache-hostname.service:
+#          [Unit]
+#          Description=Publish ccache.local hostname via Avahi
+#          After=avahi-daemon.service redis-server.service
+#          BindsTo=redis-server.service
+#          [Service]
+#          Type=simple
+#          ExecStart=/usr/bin/avahi-publish-address -R ccache.local IP_ADDRESS
+#          Restart=on-failure
+#          [Install]
+#          WantedBy=redis-server.service
+#
+#   Client requirements for mDNS resolution (one of):
+#     - libnss-resolve (systemd-resolved NSS module):
+#         apt install libnss-resolve
+#         /etc/nsswitch.conf: hosts: files resolve [!UNAVAIL=return] dns myhostname
+#     - libnss-mdns (standalone mDNS NSS module):
+#         apt install libnss-mdns
+#         /etc/nsswitch.conf: hosts: files mdns4_minimal [NOTFOUND=return] dns myhostname
+#
+# Fallback behavior:
+#   If CCACHE_REMOTE_STORAGE is not set and ccache.local is not resolvable,
+#   extension silently falls back to local ccache only.
+#
+# Cache sharing requirements:
+#   For cache to be shared across multiple build hosts, the Armbian project
+#   path must be identical on all machines (e.g., /home/build/armbian).
+#   This is because ccache includes the working directory in the cache key.
+#   Docker builds automatically use consistent paths (/armbian/...).
+
+# Default Redis connection timeout in milliseconds (can be overridden by user)
+# Note: Must be set before extension loads (e.g., via environment or command line)
+declare -g -r CCACHE_REDIS_CONNECT_TIMEOUT="${CCACHE_REDIS_CONNECT_TIMEOUT:-500}"
+
+# List of ccache environment variables to pass through to builds
+declare -g -a CCACHE_PASSTHROUGH_VARS=(
+	CCACHE_REDIS_CONNECT_TIMEOUT
+	CCACHE_BASEDIR
+	CCACHE_REMOTE_STORAGE
+	CCACHE_REMOTE_ONLY
+	CCACHE_READONLY
+	CCACHE_RECACHE
+	CCACHE_RESHARE
+	CCACHE_DISABLE
+	CCACHE_MAXSIZE
+	CCACHE_MAXFILES
+	CCACHE_NAMESPACE
+	CCACHE_SLOPPINESS
+	CCACHE_UMASK
+	CCACHE_LOGFILE
+	CCACHE_DEBUGLEVEL
+	CCACHE_STATSLOG
+	CCACHE_PCH_EXTSUM
+)
+
+# Query Redis stats (keys count and memory usage)
+function get_redis_stats() {
+	local ip="$1"
+	local port="${2:-6379}"
+	local stats=""
+
+	if command -v redis-cli &>/dev/null; then
+		local keys mem
+		keys=$(timeout 2 redis-cli -h "$ip" -p "$port" DBSIZE 2>/dev/null | grep -oE '[0-9]+' || true)
+		mem=$(timeout 2 redis-cli -h "$ip" -p "$port" INFO memory 2>/dev/null | grep "used_memory_human" | cut -d: -f2 | tr -d '[:space:]' || true)
+		if [[ -n "$keys" ]]; then
+			stats="keys=${keys:-0}, mem=${mem:-?}"
+		fi
+	else
+		# Fallback: try netcat for basic connectivity check
+		if nc -z -w 2 "$ip" "$port" 2>/dev/null; then
+			stats="reachable (redis-cli not installed for detailed stats)"
+		fi
+	fi
+	echo "$stats"
+}
+
+# This runs on the HOST just before Docker container is launched.
+# Resolves 'ccache.local' via mDNS (requires Avahi on server publishing this hostname
+# with: avahi-publish-address -R ccache.local <IP>) and passes the resolved IP
+# to Docker container via CCACHE_REMOTE_STORAGE environment variable.
+# mDNS resolution doesn't work inside Docker, so we must resolve on host.
+function host_pre_docker_launch__setup_remote_ccache() {
+	# If CCACHE_REMOTE_STORAGE not set, try to resolve ccache.local via mDNS
+	if [[ -z "${CCACHE_REMOTE_STORAGE}" ]]; then
+		local ccache_ip
+		ccache_ip=$(getent hosts ccache.local 2>/dev/null | awk '{print $1; exit}' || true)
+
+		if [[ -n "${ccache_ip}" ]]; then
+			display_alert "Remote ccache discovered on host" "redis://${ccache_ip}:6379" "info"
+
+			# Show Redis stats
+			local stats
+			stats=$(get_redis_stats "${ccache_ip}" 6379)
+			if [[ -n "$stats" ]]; then
+				display_alert "Remote ccache stats" "${stats}" "info"
+			fi
+
+			export CCACHE_REMOTE_STORAGE="redis://${ccache_ip}:6379|connect-timeout=${CCACHE_REDIS_CONNECT_TIMEOUT}"
+		else
+			display_alert "Remote ccache not found on host" "ccache.local not resolvable via mDNS" "debug"
+		fi
+	else
+		display_alert "Remote ccache pre-configured" "${CCACHE_REMOTE_STORAGE}" "info"
+	fi
+
+	# Pass all set CCACHE_* variables to Docker
+	local var val
+	for var in "${CCACHE_PASSTHROUGH_VARS[@]}"; do
+		val="${!var}"
+		if [[ -n "${val}" ]]; then
+			DOCKER_EXTRA_ARGS+=("--env" "${var}=${val}")
+			display_alert "Docker env" "${var}=${val}" "debug"
+		fi
+	done
+}
+
+# Hook: Show ccache remote storage statistics after each compilation (kernel, uboot)
+function ccache_post_compilation__show_remote_stats() {
+	if [[ -n "${CCACHE_REMOTE_STORAGE}" ]]; then
+		local stats_output total pct
+		local read_hit read_miss write error
+		stats_output=$(ccache --print-stats 2>&1 || true)
+		read_hit=$(echo "$stats_output" | grep "^remote_storage_read_hit" | cut -f2 || true)
+		read_miss=$(echo "$stats_output" | grep "^remote_storage_read_miss" | cut -f2 || true)
+		write=$(echo "$stats_output" | grep "^remote_storage_write" | cut -f2 || true)
+		error=$(echo "$stats_output" | grep "^remote_storage_error" | cut -f2 || true)
+		# Ensure numeric values for arithmetic (grep may return empty string)
+		[[ "${read_hit}" =~ ^[0-9]+$ ]] || read_hit=0
+		[[ "${read_miss}" =~ ^[0-9]+$ ]] || read_miss=0
+		[[ "${write}" =~ ^[0-9]+$ ]] || write=0
+		[[ "${error}" =~ ^[0-9]+$ ]] || error=0
+		total=$(( read_hit + read_miss ))
+		pct=0
+		if [[ $total -gt 0 ]]; then
+			pct=$(( read_hit * 100 / total ))
+		fi
+		display_alert "Remote ccache result" "hit=${read_hit} miss=${read_miss} write=${write} err=${error} (${pct}%)" "info"
+	fi
+}
+
+# This runs inside Docker (or native build) during configuration
+function extension_prepare_config__setup_remote_ccache() {
+	# Enable ccache
+	declare -g USE_CCACHE=yes
+
+	# If CCACHE_REMOTE_STORAGE not set, try to resolve ccache.local via mDNS
+	if [[ -z "${CCACHE_REMOTE_STORAGE}" ]]; then
+		local ccache_ip
+		ccache_ip=$(getent hosts ccache.local 2>/dev/null | awk '{print $1; exit}' || true)
+
+		if [[ -n "${ccache_ip}" ]]; then
+			export CCACHE_REMOTE_STORAGE="redis://${ccache_ip}:6379|connect-timeout=${CCACHE_REDIS_CONNECT_TIMEOUT}"
+			display_alert "Remote ccache discovered" "${CCACHE_REMOTE_STORAGE}" "info"
+		else
+			display_alert "Remote ccache not available" "using local cache only" "debug"
+		fi
+	else
+		display_alert "Remote ccache configured" "${CCACHE_REMOTE_STORAGE}" "info"
+	fi
+
+	return 0
+}
+
+# This hook runs right before kernel make - add ccache env vars to make environment.
+# Required because kernel build uses 'env -i' which clears all environment variables.
+function kernel_make_config__add_ccache_remote_storage() {
+	local var val
+	for var in "${CCACHE_PASSTHROUGH_VARS[@]}"; do
+		val="${!var}"
+		if [[ -n "${val}" ]]; then
+			common_make_envs+=("${var}=${val@Q}")
+			display_alert "Kernel make: ${var}" "${val}" "debug"
+		fi
+	done
+}
+
+# This hook runs right before u-boot make - add ccache env vars to make environment.
+# Required because u-boot build uses 'env -i' which clears all environment variables.
+function uboot_make_config__add_ccache_remote_storage() {
+	local var val
+	for var in "${CCACHE_PASSTHROUGH_VARS[@]}"; do
+		val="${!var}"
+		if [[ -n "${val}" ]]; then
+			uboot_make_envs+=("${var}=${val@Q}")
+			display_alert "U-boot make: ${var}" "${val}" "debug"
+		fi
+	done
+}

lib/functions/compilation/ccache.sh

@@ -7,6 +7,29 @@
 # This file is a part of the Armbian Build Framework
 # https://github.com/armbian/build/
 
+# Helper function to show ccache stats - used as cleanup handler for interruption case
+function ccache_show_compilation_stats() {
+	local stats_output direct_hit direct_miss total pct
+	stats_output=$(ccache --print-stats 2>&1 || true)
+	direct_hit=$(echo "$stats_output" | grep "^direct_cache_hit" | cut -f2 || true)
+	direct_miss=$(echo "$stats_output" | grep "^direct_cache_miss" | cut -f2 || true)
+	# Ensure numeric values for arithmetic (grep may return empty string)
+	[[ "${direct_hit}" =~ ^[0-9]+$ ]] || direct_hit=0
+	[[ "${direct_miss}" =~ ^[0-9]+$ ]] || direct_miss=0
+	total=$(( direct_hit + direct_miss ))
+	pct=0
+	if [[ $total -gt 0 ]]; then
+		pct=$(( direct_hit * 100 / total ))
+	fi
+	display_alert "Ccache result" "hit=${direct_hit} miss=${direct_miss} (${pct}%)" "info"
+
+	# Hook for extensions to show additional stats (e.g., remote storage)
+	call_extension_method "ccache_post_compilation" <<- 'CCACHE_POST_COMPILATION'
+		*called after ccache-wrapped compilation completes (success or failure)*
+		Useful for displaying remote cache statistics or other post-build info.
+	CCACHE_POST_COMPILATION
+}
+
 function do_with_ccache_statistics() {
 
 	display_alert "Clearing ccache statistics" "ccache" "ccache"
@@ -35,8 +58,20 @@ function do_with_ccache_statistics() {
 		run_host_command_logged ccache --show-config "&&" sync
 	fi
 
+	# Register cleanup handler to show stats even if build is interrupted
+	add_cleanup_handler ccache_show_compilation_stats
+
 	display_alert "Running ccache'd build..." "ccache" "ccache"
-	"$@"
+	local build_exit_code=0
+	"$@" || build_exit_code=$?
+
+	# Show stats and remove from cleanup handlers (so it doesn't run twice on exit)
+	execute_and_remove_cleanup_handler ccache_show_compilation_stats
+
+	# Re-raise the error if the build failed
+	if [[ ${build_exit_code} -ne 0 ]]; then
+		return ${build_exit_code}
+	fi
 
 	if [[ "${SHOW_CCACHE}" == "yes" ]]; then
 		display_alert "Display ccache statistics" "ccache" "ccache"

lib/functions/compilation/kernel-make.sh

@@ -31,7 +31,7 @@ function run_kernel_make_internal() {
 
 	# If CCACHE_DIR is set, pass it to the kernel build; Pass the ccache dir explicitly, since we'll run under "env -i"
 	if [[ -n "${CCACHE_DIR}" ]]; then
-		common_make_envs+=("CCACHE_DIR='${CCACHE_DIR}'")
+		common_make_envs+=("CCACHE_DIR=${CCACHE_DIR@Q}")
 	fi
 
 	# Add the distcc envs, if any.
@@ -74,6 +74,16 @@ function run_kernel_make_internal() {
 		common_make_params_quoted+=("${llvm_flag}")
 	fi
 
+	call_extension_method "kernel_make_config" <<- 'KERNEL_MAKE_CONFIG'
+		*Hook to customize kernel make environment and parameters*
+		Called right before invoking make for kernel compilation.
+		Available arrays to modify:
+		  - common_make_envs[@]: environment variables passed via "env -i" (e.g., CCACHE_REMOTE_STORAGE)
+		  - common_make_params_quoted[@]: make command parameters (e.g., custom flags)
+		Available read-only variables:
+		  - KERNEL_COMPILER, ARCHITECTURE, BRANCH, LINUXFAMILY, toolchain
+	KERNEL_MAKE_CONFIG
+
 	# Allow extensions to modify make parameters and environment variables
 	call_extension_method "custom_kernel_make_params" <<- 'CUSTOM_KERNEL_MAKE_PARAMS'
 		*Customize kernel make parameters before compilation*

lib/functions/compilation/uboot.sh

@@ -253,6 +253,27 @@ function compile_uboot_target() {
 		"PYTHONPATH=\"${PYTHON3_INFO[MODULES_PATH]}:${PYTHONPATH}\"" # Insert the pip modules downloaded by Armbian into PYTHONPATH (needed e.g. for pyelftools)
 	)
 
+	# Pass the ccache directories explicitly, since we'll run under "env -i"
+	if [[ -n "${CCACHE_DIR}" ]]; then
+		uboot_make_envs+=("CCACHE_DIR=${CCACHE_DIR@Q}")
+	fi
+	if [[ -n "${CCACHE_TEMPDIR}" ]]; then
+		uboot_make_envs+=("CCACHE_TEMPDIR=${CCACHE_TEMPDIR@Q}")
+	fi
+
+	# workaround when two compilers are needed
+	cross_compile="CROSS_COMPILE=\"$CCACHE $UBOOT_COMPILER\""
+	# When UBOOT_TOOLCHAIN2 is set, the board's uboot_custom_postprocess handles compilers;
+	# pass a harmless dummy env var since empty make parameters cause errors
+	[[ -n $UBOOT_TOOLCHAIN2 ]] && cross_compile="ARMBIAN=foe"
+
+	call_extension_method "uboot_make_config" <<- 'UBOOT_MAKE_CONFIG'
+		*Hook to customize u-boot make environment*
+		Called right before invoking make for u-boot compilation.
+		Available array to modify:
+		  - uboot_make_envs[@]: environment variables passed via "env -i" (e.g., CCACHE_REMOTE_STORAGE)
+	UBOOT_MAKE_CONFIG
+
 	display_alert "${uboot_prefix}Compiling u-boot" "${version} ${target_make} with gcc '${gcc_version_main}'" "info"
 	declare -g if_error_detail_message="${uboot_prefix}Failed to build u-boot ${version} ${target_make}"
 	do_with_ccache_statistics run_host_command_logged_long_running \

lib/functions/configuration/compilation-config.sh

@@ -17,6 +17,11 @@ function prepare_compilation_vars() {
 		# private ccache directory to avoid permission issues when using build script with "sudo"
 		# see https://ccache.samba.org/manual.html#_sharing_a_cache for alternative solution
 		[[ $PRIVATE_CCACHE == yes ]] && export CCACHE_DIR=$SRC/cache/ccache # actual export
+
+		# Set default umask for ccache to allow write access for all users (enables cache sharing)
+		# CCACHE_UMASK=000 creates files with permissions 666 (rw-rw-rw-) and dirs with 777 (rwxrwxrwx)
+		# Only set this for shared cache, not for private cache
+		[[ -z "${CCACHE_UMASK}" && "${PRIVATE_CCACHE}" != "yes" ]] && export CCACHE_UMASK=000
 	else
 		CCACHE=""
 	fi

lib/functions/host/docker.sh

@@ -568,8 +568,23 @@ function docker_cli_prepare_launch() {
 		display_alert "Not running in a terminal" "not passing through stdin to Docker" "debug"
 	fi
 
-	# if DOCKER_EXTRA_ARGS is an array and has more than zero elements, add its contents to the DOCKER_ARGS array
-	if [[ "${DOCKER_EXTRA_ARGS[*]+isset}" == "isset" && "${#DOCKER_EXTRA_ARGS[@]}" -gt 0 ]]; then
+	# Initialize DOCKER_EXTRA_ARGS for extensions to populate
+	declare -g -a DOCKER_EXTRA_ARGS=()
+
+	# Hook for extensions to add Docker arguments before launch
+	call_extension_method "host_pre_docker_launch" <<- 'HOST_PRE_DOCKER_LAUNCH'
+		*run on host just before Docker container is launched*
+		Extensions can add Docker arguments by appending to DOCKER_EXTRA_ARGS array.
+		Each array element should be a complete argument (e.g., "--env", "MY_VAR=value" as separate elements).
+		Example: DOCKER_EXTRA_ARGS+=("--env" "MY_VAR=value" "--mount" "type=bind,src=/a,dst=/b")
+		Available variables:
+		  - DOCKER_ARGS[@]: current Docker arguments (do not modify directly)
+		  - DOCKER_EXTRA_ARGS[@]: array to append extra arguments for docker run
+		  - DOCKER_ARMBIAN_TARGET_PATH: path inside container (/armbian)
+	HOST_PRE_DOCKER_LAUNCH
+
+	# Add DOCKER_EXTRA_ARGS to DOCKER_ARGS if any were added by extensions
+	if [[ "${#DOCKER_EXTRA_ARGS[@]}" -gt 0 ]]; then
 		display_alert "Adding extra Docker arguments" "${DOCKER_EXTRA_ARGS[*]}" "debug"
 		DOCKER_ARGS+=("${DOCKER_EXTRA_ARGS[@]}")
 	fi