diff --git a/extensions/ccache-remote.sh b/extensions/ccache-remote.sh new file mode 100644 index 000000000000..b2e4f9e531bd --- /dev/null +++ b/extensions/ccache-remote.sh @@ -0,0 +1,251 @@ +# Extension: ccache-remote +# Enables ccache with remote Redis storage for sharing compilation cache across build hosts +# +# Documentation: https://ccache.dev/howto/redis-storage.html +# See also: https://ccache.dev/manual/4.10.html#config_remote_storage +# +# Usage: +# # With Avahi/mDNS auto-discovery: +# ./compile.sh ENABLE_EXTENSIONS=ccache-remote BOARD=... +# +# # With explicit Redis server (no Avahi needed): +# ./compile.sh ENABLE_EXTENSIONS=ccache-remote CCACHE_REMOTE_STORAGE="redis://192.168.1.65:6379" BOARD=... +# +# # Disable local cache, use remote only (saves local disk space): +# ./compile.sh ENABLE_EXTENSIONS=ccache-remote CCACHE_REMOTE_ONLY=yes BOARD=... +# +# Automatically sets USE_CCACHE=yes +# +# Supported ccache environment variables (passed through to builds): +# See: https://ccache.dev/manual/latest.html#_configuration_options +# CCACHE_BASEDIR - base directory for path normalization (enables cache sharing) +# CCACHE_REMOTE_STORAGE - remote storage URL (redis://...) +# CCACHE_REMOTE_ONLY - use only remote storage, disable local cache +# CCACHE_READONLY - read-only mode, don't update cache +# CCACHE_RECACHE - don't use cached results, but update cache +# CCACHE_RESHARE - rewrite cache entries to remote storage +# CCACHE_DISABLE - disable ccache completely +# CCACHE_MAXSIZE - maximum cache size (e.g., "10G") +# CCACHE_MAXFILES - maximum number of files in cache +# CCACHE_NAMESPACE - cache namespace for isolation +# CCACHE_SLOPPINESS - comma-separated list of sloppiness options +# CCACHE_UMASK - umask for cache files +# CCACHE_LOGFILE - path to log file +# CCACHE_DEBUGLEVEL - debug level (1-2) +# CCACHE_STATSLOG - path to stats log file +# CCACHE_PCH_EXTSUM - include PCH extension in hash +# +# CCACHE_REMOTE_STORAGE format (ccache 4.4+): +# redis://HOST[:PORT][|attribute=value...] +# Common attributes: +# connect-timeout=N - connection timeout in milliseconds (default: 100) +# operation-timeout=N - operation timeout in milliseconds (default: 10000) +# Example: "redis://192.168.1.65:6379|connect-timeout=500" +# +# Avahi/mDNS auto-discovery: +# This extension tries to resolve 'ccache.local' hostname via mDNS. +# To publish this hostname on Redis server, run: +# avahi-publish-address -R ccache.local +# Or create a systemd service (see below). +# +# Server setup example: +# 1. Install: apt install redis-server avahi-daemon avahi-utils +# 2. Configure Redis (/etc/redis/redis.conf): +# bind 0.0.0.0 :: +# protected-mode no +# maxmemory 4G +# maxmemory-policy allkeys-lru +# WARNING: This configuration is INSECURE - Redis is open without authentication. +# Use ONLY in a fully trusted private network with no internet access. +# For secure setup (password, TLS, ACL), see: https://redis.io/docs/management/security/ +# 3. Publish hostname (replace IP_ADDRESS with actual IP): +# avahi-publish-address -R ccache.local IP_ADDRESS +# Or as systemd service /etc/systemd/system/ccache-hostname.service: +# [Unit] +# Description=Publish ccache.local hostname via Avahi +# After=avahi-daemon.service redis-server.service +# BindsTo=redis-server.service +# [Service] +# Type=simple +# ExecStart=/usr/bin/avahi-publish-address -R ccache.local IP_ADDRESS +# Restart=on-failure +# [Install] +# WantedBy=redis-server.service +# +# Client requirements for mDNS resolution (one of): +# - libnss-resolve (systemd-resolved NSS module): +# apt install libnss-resolve +# /etc/nsswitch.conf: hosts: files resolve [!UNAVAIL=return] dns myhostname +# - libnss-mdns (standalone mDNS NSS module): +# apt install libnss-mdns +# /etc/nsswitch.conf: hosts: files mdns4_minimal [NOTFOUND=return] dns myhostname +# +# Fallback behavior: +# If CCACHE_REMOTE_STORAGE is not set and ccache.local is not resolvable, +# extension silently falls back to local ccache only. +# +# Cache sharing requirements: +# For cache to be shared across multiple build hosts, the Armbian project +# path must be identical on all machines (e.g., /home/build/armbian). +# This is because ccache includes the working directory in the cache key. +# Docker builds automatically use consistent paths (/armbian/...). + +# Default Redis connection timeout in milliseconds (can be overridden by user) +# Note: Must be set before extension loads (e.g., via environment or command line) +declare -g -r CCACHE_REDIS_CONNECT_TIMEOUT="${CCACHE_REDIS_CONNECT_TIMEOUT:-500}" + +# List of ccache environment variables to pass through to builds +declare -g -a CCACHE_PASSTHROUGH_VARS=( + CCACHE_REDIS_CONNECT_TIMEOUT + CCACHE_BASEDIR + CCACHE_REMOTE_STORAGE + CCACHE_REMOTE_ONLY + CCACHE_READONLY + CCACHE_RECACHE + CCACHE_RESHARE + CCACHE_DISABLE + CCACHE_MAXSIZE + CCACHE_MAXFILES + CCACHE_NAMESPACE + CCACHE_SLOPPINESS + CCACHE_UMASK + CCACHE_LOGFILE + CCACHE_DEBUGLEVEL + CCACHE_STATSLOG + CCACHE_PCH_EXTSUM +) + +# Query Redis stats (keys count and memory usage) +function get_redis_stats() { + local ip="$1" + local port="${2:-6379}" + local stats="" + + if command -v redis-cli &>/dev/null; then + local keys mem + keys=$(timeout 2 redis-cli -h "$ip" -p "$port" DBSIZE 2>/dev/null | grep -oE '[0-9]+' || true) + mem=$(timeout 2 redis-cli -h "$ip" -p "$port" INFO memory 2>/dev/null | grep "used_memory_human" | cut -d: -f2 | tr -d '[:space:]' || true) + if [[ -n "$keys" ]]; then + stats="keys=${keys:-0}, mem=${mem:-?}" + fi + else + # Fallback: try netcat for basic connectivity check + if nc -z -w 2 "$ip" "$port" 2>/dev/null; then + stats="reachable (redis-cli not installed for detailed stats)" + fi + fi + echo "$stats" +} + +# This runs on the HOST just before Docker container is launched. +# Resolves 'ccache.local' via mDNS (requires Avahi on server publishing this hostname +# with: avahi-publish-address -R ccache.local ) and passes the resolved IP +# to Docker container via CCACHE_REMOTE_STORAGE environment variable. +# mDNS resolution doesn't work inside Docker, so we must resolve on host. +function host_pre_docker_launch__setup_remote_ccache() { + # If CCACHE_REMOTE_STORAGE not set, try to resolve ccache.local via mDNS + if [[ -z "${CCACHE_REMOTE_STORAGE}" ]]; then + local ccache_ip + ccache_ip=$(getent hosts ccache.local 2>/dev/null | awk '{print $1; exit}' || true) + + if [[ -n "${ccache_ip}" ]]; then + display_alert "Remote ccache discovered on host" "redis://${ccache_ip}:6379" "info" + + # Show Redis stats + local stats + stats=$(get_redis_stats "${ccache_ip}" 6379) + if [[ -n "$stats" ]]; then + display_alert "Remote ccache stats" "${stats}" "info" + fi + + export CCACHE_REMOTE_STORAGE="redis://${ccache_ip}:6379|connect-timeout=${CCACHE_REDIS_CONNECT_TIMEOUT}" + else + display_alert "Remote ccache not found on host" "ccache.local not resolvable via mDNS" "debug" + fi + else + display_alert "Remote ccache pre-configured" "${CCACHE_REMOTE_STORAGE}" "info" + fi + + # Pass all set CCACHE_* variables to Docker + local var val + for var in "${CCACHE_PASSTHROUGH_VARS[@]}"; do + val="${!var}" + if [[ -n "${val}" ]]; then + DOCKER_EXTRA_ARGS+=("--env" "${var}=${val}") + display_alert "Docker env" "${var}=${val}" "debug" + fi + done +} + +# Hook: Show ccache remote storage statistics after each compilation (kernel, uboot) +function ccache_post_compilation__show_remote_stats() { + if [[ -n "${CCACHE_REMOTE_STORAGE}" ]]; then + local stats_output total pct + local read_hit read_miss write error + stats_output=$(ccache --print-stats 2>&1 || true) + read_hit=$(echo "$stats_output" | grep "^remote_storage_read_hit" | cut -f2 || true) + read_miss=$(echo "$stats_output" | grep "^remote_storage_read_miss" | cut -f2 || true) + write=$(echo "$stats_output" | grep "^remote_storage_write" | cut -f2 || true) + error=$(echo "$stats_output" | grep "^remote_storage_error" | cut -f2 || true) + # Ensure numeric values for arithmetic (grep may return empty string) + [[ "${read_hit}" =~ ^[0-9]+$ ]] || read_hit=0 + [[ "${read_miss}" =~ ^[0-9]+$ ]] || read_miss=0 + [[ "${write}" =~ ^[0-9]+$ ]] || write=0 + [[ "${error}" =~ ^[0-9]+$ ]] || error=0 + total=$(( read_hit + read_miss )) + pct=0 + if [[ $total -gt 0 ]]; then + pct=$(( read_hit * 100 / total )) + fi + display_alert "Remote ccache result" "hit=${read_hit} miss=${read_miss} write=${write} err=${error} (${pct}%)" "info" + fi +} + +# This runs inside Docker (or native build) during configuration +function extension_prepare_config__setup_remote_ccache() { + # Enable ccache + declare -g USE_CCACHE=yes + + # If CCACHE_REMOTE_STORAGE not set, try to resolve ccache.local via mDNS + if [[ -z "${CCACHE_REMOTE_STORAGE}" ]]; then + local ccache_ip + ccache_ip=$(getent hosts ccache.local 2>/dev/null | awk '{print $1; exit}' || true) + + if [[ -n "${ccache_ip}" ]]; then + export CCACHE_REMOTE_STORAGE="redis://${ccache_ip}:6379|connect-timeout=${CCACHE_REDIS_CONNECT_TIMEOUT}" + display_alert "Remote ccache discovered" "${CCACHE_REMOTE_STORAGE}" "info" + else + display_alert "Remote ccache not available" "using local cache only" "debug" + fi + else + display_alert "Remote ccache configured" "${CCACHE_REMOTE_STORAGE}" "info" + fi + + return 0 +} + +# This hook runs right before kernel make - add ccache env vars to make environment. +# Required because kernel build uses 'env -i' which clears all environment variables. +function kernel_make_config__add_ccache_remote_storage() { + local var val + for var in "${CCACHE_PASSTHROUGH_VARS[@]}"; do + val="${!var}" + if [[ -n "${val}" ]]; then + common_make_envs+=("${var}=${val@Q}") + display_alert "Kernel make: ${var}" "${val}" "debug" + fi + done +} + +# This hook runs right before u-boot make - add ccache env vars to make environment. +# Required because u-boot build uses 'env -i' which clears all environment variables. +function uboot_make_config__add_ccache_remote_storage() { + local var val + for var in "${CCACHE_PASSTHROUGH_VARS[@]}"; do + val="${!var}" + if [[ -n "${val}" ]]; then + uboot_make_envs+=("${var}=${val@Q}") + display_alert "U-boot make: ${var}" "${val}" "debug" + fi + done +} diff --git a/lib/functions/compilation/ccache.sh b/lib/functions/compilation/ccache.sh index 43f854542a64..b2b8e117ac52 100644 --- a/lib/functions/compilation/ccache.sh +++ b/lib/functions/compilation/ccache.sh @@ -7,6 +7,29 @@ # This file is a part of the Armbian Build Framework # https://github.com/armbian/build/ +# Helper function to show ccache stats - used as cleanup handler for interruption case +function ccache_show_compilation_stats() { + local stats_output direct_hit direct_miss total pct + stats_output=$(ccache --print-stats 2>&1 || true) + direct_hit=$(echo "$stats_output" | grep "^direct_cache_hit" | cut -f2 || true) + direct_miss=$(echo "$stats_output" | grep "^direct_cache_miss" | cut -f2 || true) + # Ensure numeric values for arithmetic (grep may return empty string) + [[ "${direct_hit}" =~ ^[0-9]+$ ]] || direct_hit=0 + [[ "${direct_miss}" =~ ^[0-9]+$ ]] || direct_miss=0 + total=$(( direct_hit + direct_miss )) + pct=0 + if [[ $total -gt 0 ]]; then + pct=$(( direct_hit * 100 / total )) + fi + display_alert "Ccache result" "hit=${direct_hit} miss=${direct_miss} (${pct}%)" "info" + + # Hook for extensions to show additional stats (e.g., remote storage) + call_extension_method "ccache_post_compilation" <<- 'CCACHE_POST_COMPILATION' + *called after ccache-wrapped compilation completes (success or failure)* + Useful for displaying remote cache statistics or other post-build info. + CCACHE_POST_COMPILATION +} + function do_with_ccache_statistics() { display_alert "Clearing ccache statistics" "ccache" "ccache" @@ -35,8 +58,20 @@ function do_with_ccache_statistics() { run_host_command_logged ccache --show-config "&&" sync fi + # Register cleanup handler to show stats even if build is interrupted + add_cleanup_handler ccache_show_compilation_stats + display_alert "Running ccache'd build..." "ccache" "ccache" - "$@" + local build_exit_code=0 + "$@" || build_exit_code=$? + + # Show stats and remove from cleanup handlers (so it doesn't run twice on exit) + execute_and_remove_cleanup_handler ccache_show_compilation_stats + + # Re-raise the error if the build failed + if [[ ${build_exit_code} -ne 0 ]]; then + return ${build_exit_code} + fi if [[ "${SHOW_CCACHE}" == "yes" ]]; then display_alert "Display ccache statistics" "ccache" "ccache" diff --git a/lib/functions/compilation/kernel-make.sh b/lib/functions/compilation/kernel-make.sh index 6063d58a672b..a9b95bc0dc45 100644 --- a/lib/functions/compilation/kernel-make.sh +++ b/lib/functions/compilation/kernel-make.sh @@ -31,7 +31,7 @@ function run_kernel_make_internal() { # If CCACHE_DIR is set, pass it to the kernel build; Pass the ccache dir explicitly, since we'll run under "env -i" if [[ -n "${CCACHE_DIR}" ]]; then - common_make_envs+=("CCACHE_DIR='${CCACHE_DIR}'") + common_make_envs+=("CCACHE_DIR=${CCACHE_DIR@Q}") fi # Add the distcc envs, if any. @@ -74,6 +74,16 @@ function run_kernel_make_internal() { common_make_params_quoted+=("${llvm_flag}") fi + call_extension_method "kernel_make_config" <<- 'KERNEL_MAKE_CONFIG' + *Hook to customize kernel make environment and parameters* + Called right before invoking make for kernel compilation. + Available arrays to modify: + - common_make_envs[@]: environment variables passed via "env -i" (e.g., CCACHE_REMOTE_STORAGE) + - common_make_params_quoted[@]: make command parameters (e.g., custom flags) + Available read-only variables: + - KERNEL_COMPILER, ARCHITECTURE, BRANCH, LINUXFAMILY, toolchain + KERNEL_MAKE_CONFIG + # Allow extensions to modify make parameters and environment variables call_extension_method "custom_kernel_make_params" <<- 'CUSTOM_KERNEL_MAKE_PARAMS' *Customize kernel make parameters before compilation* diff --git a/lib/functions/compilation/uboot.sh b/lib/functions/compilation/uboot.sh index a6be864605ba..d2ffc103938d 100644 --- a/lib/functions/compilation/uboot.sh +++ b/lib/functions/compilation/uboot.sh @@ -253,6 +253,27 @@ function compile_uboot_target() { "PYTHONPATH=\"${PYTHON3_INFO[MODULES_PATH]}:${PYTHONPATH}\"" # Insert the pip modules downloaded by Armbian into PYTHONPATH (needed e.g. for pyelftools) ) + # Pass the ccache directories explicitly, since we'll run under "env -i" + if [[ -n "${CCACHE_DIR}" ]]; then + uboot_make_envs+=("CCACHE_DIR=${CCACHE_DIR@Q}") + fi + if [[ -n "${CCACHE_TEMPDIR}" ]]; then + uboot_make_envs+=("CCACHE_TEMPDIR=${CCACHE_TEMPDIR@Q}") + fi + + # workaround when two compilers are needed + cross_compile="CROSS_COMPILE=\"$CCACHE $UBOOT_COMPILER\"" + # When UBOOT_TOOLCHAIN2 is set, the board's uboot_custom_postprocess handles compilers; + # pass a harmless dummy env var since empty make parameters cause errors + [[ -n $UBOOT_TOOLCHAIN2 ]] && cross_compile="ARMBIAN=foe" + + call_extension_method "uboot_make_config" <<- 'UBOOT_MAKE_CONFIG' + *Hook to customize u-boot make environment* + Called right before invoking make for u-boot compilation. + Available array to modify: + - uboot_make_envs[@]: environment variables passed via "env -i" (e.g., CCACHE_REMOTE_STORAGE) + UBOOT_MAKE_CONFIG + display_alert "${uboot_prefix}Compiling u-boot" "${version} ${target_make} with gcc '${gcc_version_main}'" "info" declare -g if_error_detail_message="${uboot_prefix}Failed to build u-boot ${version} ${target_make}" do_with_ccache_statistics run_host_command_logged_long_running \ diff --git a/lib/functions/configuration/compilation-config.sh b/lib/functions/configuration/compilation-config.sh index f917689d975b..dfdaec04aea4 100644 --- a/lib/functions/configuration/compilation-config.sh +++ b/lib/functions/configuration/compilation-config.sh @@ -17,6 +17,11 @@ function prepare_compilation_vars() { # private ccache directory to avoid permission issues when using build script with "sudo" # see https://ccache.samba.org/manual.html#_sharing_a_cache for alternative solution [[ $PRIVATE_CCACHE == yes ]] && export CCACHE_DIR=$SRC/cache/ccache # actual export + + # Set default umask for ccache to allow write access for all users (enables cache sharing) + # CCACHE_UMASK=000 creates files with permissions 666 (rw-rw-rw-) and dirs with 777 (rwxrwxrwx) + # Only set this for shared cache, not for private cache + [[ -z "${CCACHE_UMASK}" && "${PRIVATE_CCACHE}" != "yes" ]] && export CCACHE_UMASK=000 else CCACHE="" fi diff --git a/lib/functions/host/docker.sh b/lib/functions/host/docker.sh index 6b1e394a4abf..9f86c736fa73 100644 --- a/lib/functions/host/docker.sh +++ b/lib/functions/host/docker.sh @@ -568,8 +568,23 @@ function docker_cli_prepare_launch() { display_alert "Not running in a terminal" "not passing through stdin to Docker" "debug" fi - # if DOCKER_EXTRA_ARGS is an array and has more than zero elements, add its contents to the DOCKER_ARGS array - if [[ "${DOCKER_EXTRA_ARGS[*]+isset}" == "isset" && "${#DOCKER_EXTRA_ARGS[@]}" -gt 0 ]]; then + # Initialize DOCKER_EXTRA_ARGS for extensions to populate + declare -g -a DOCKER_EXTRA_ARGS=() + + # Hook for extensions to add Docker arguments before launch + call_extension_method "host_pre_docker_launch" <<- 'HOST_PRE_DOCKER_LAUNCH' + *run on host just before Docker container is launched* + Extensions can add Docker arguments by appending to DOCKER_EXTRA_ARGS array. + Each array element should be a complete argument (e.g., "--env", "MY_VAR=value" as separate elements). + Example: DOCKER_EXTRA_ARGS+=("--env" "MY_VAR=value" "--mount" "type=bind,src=/a,dst=/b") + Available variables: + - DOCKER_ARGS[@]: current Docker arguments (do not modify directly) + - DOCKER_EXTRA_ARGS[@]: array to append extra arguments for docker run + - DOCKER_ARMBIAN_TARGET_PATH: path inside container (/armbian) + HOST_PRE_DOCKER_LAUNCH + + # Add DOCKER_EXTRA_ARGS to DOCKER_ARGS if any were added by extensions + if [[ "${#DOCKER_EXTRA_ARGS[@]}" -gt 0 ]]; then display_alert "Adding extra Docker arguments" "${DOCKER_EXTRA_ARGS[*]}" "debug" DOCKER_ARGS+=("${DOCKER_EXTRA_ARGS[@]}") fi