diff --git a/.github/actions/godot-cache-restore/action.yml b/.github/actions/godot-cache-restore/action.yml index f10222bf5..9eac8eecf 100644 --- a/.github/actions/godot-cache-restore/action.yml +++ b/.github/actions/godot-cache-restore/action.yml @@ -6,7 +6,7 @@ inputs: default: ${{ github.job }} scons-cache: description: The SCons cache path. - default: ${{ github.workspace }}/.scons-cache/ + default: ${{ github.workspace }}/.scons_cache/ runs: using: composite @@ -18,7 +18,6 @@ runs: key: ${{ inputs.cache-name }}-${{ env.GODOT_BASE_BRANCH }}-${{ github.ref }}-${{ github.sha }} restore-keys: | - ${{ inputs.cache-name }}-${{ env.GODOT_BASE_BRANCH }}-${{ github.ref }}-${{ github.sha }} ${{ inputs.cache-name }}-${{ env.GODOT_BASE_BRANCH }}-${{ github.ref }} ${{ inputs.cache-name }}-${{ env.GODOT_BASE_BRANCH }}-refs/heads/${{ env.GODOT_BASE_BRANCH }} ${{ inputs.cache-name }}-${{ env.GODOT_BASE_BRANCH }} diff --git a/.github/actions/godot-cache-save/action.yml b/.github/actions/godot-cache-save/action.yml index df877cec6..42aa83640 100644 --- a/.github/actions/godot-cache-save/action.yml +++ b/.github/actions/godot-cache-save/action.yml @@ -6,7 +6,7 @@ inputs: default: ${{ github.job }} scons-cache: description: The SCons cache path. - default: ${{ github.workspace }}/.scons-cache/ + default: ${{ github.workspace }}/.scons_cache/ runs: using: composite diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6580de90f..b19d8d775 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,6 @@ jobs: cache-name: web-wasm32 env: - SCONS_CACHE: ${{ github.workspace }}/.scons-cache/ EM_VERSION: 3.1.39 steps: @@ -116,22 +115,22 @@ jobs: - name: Generate godot-cpp sources only run: | - scons platform=${{ matrix.platform }} verbose=yes build_library=no ${{ matrix.flags }} + scons platform=${{ matrix.platform }} verbose=yes build_library=no ${{ matrix.flags }} "cache_path=${{ github.workspace }}/.scons_cache" scons -c - name: Build godot-cpp (debug) run: | - scons platform=${{ matrix.platform }} verbose=yes target=template_debug ${{ matrix.flags }} + scons platform=${{ matrix.platform }} verbose=yes target=template_debug ${{ matrix.flags }} "cache_path=${{ github.workspace }}/.scons_cache" - name: Build test without rebuilding godot-cpp (debug) run: | cd test - scons platform=${{ matrix.platform }} verbose=yes target=template_debug ${{ matrix.flags }} build_library=no + scons platform=${{ matrix.platform }} verbose=yes target=template_debug ${{ matrix.flags }} "cache_path=${{ github.workspace }}/.scons_cache" build_library=no - name: Build test and godot-cpp (release) run: | cd test - scons platform=${{ matrix.platform }} verbose=yes target=template_release ${{ matrix.flags }} + scons platform=${{ matrix.platform }} verbose=yes target=template_release ${{ matrix.flags }} "cache_path=${{ github.workspace }}/.scons_cache" cache_limit=1 - name: Save Godot build cache uses: ./.github/actions/godot-cache-save diff --git a/.gitignore b/.gitignore index 9bb08ed34..c45a68d1b 100644 --- a/.gitignore +++ b/.gitignore @@ -196,6 +196,9 @@ compile_commands.json .venv venv +# Python modules +.*_cache/ + # Clion Configuration .idea/ cmake-build*/ diff --git a/SConstruct b/SConstruct index 8acea2624..d6e978a9a 100644 --- a/SConstruct +++ b/SConstruct @@ -40,11 +40,6 @@ if unknown: for item in unknown.items(): print(" " + item[0] + "=" + item[1]) -scons_cache_path = os.environ.get("SCONS_CACHE") -if scons_cache_path is not None: - CacheDir(scons_cache_path) - Decider("MD5") - cpp_tool.generate(env) library = env.GodotCPP() diff --git a/tools/godotcpp.py b/tools/godotcpp.py index 77a0740fc..73ff05594 100644 --- a/tools/godotcpp.py +++ b/tools/godotcpp.py @@ -337,6 +337,16 @@ def options(opts, env): opts.Add(BoolVariable("debug_symbols", "Build with debugging symbols", True)) opts.Add(BoolVariable("dev_build", "Developer build with dev-only debugging code (DEV_ENABLED)", False)) opts.Add(BoolVariable("verbose", "Enable verbose output for the compilation", False)) + opts.Add( + "cache_path", "Path to a directory where SCons cache files will be stored. No value disables the cache.", "" + ) + opts.Add( + ( + ["cache_limit", "cache_limit_gb", "cache_limit_gib"], + "Max size (in GiB) for the SCons cache. 0 means no limit.", + "0", + ) + ) # Add platform options (custom tools can override platforms) for pl in sorted(set(platforms + custom_platforms)): @@ -390,7 +400,141 @@ def make_doc_source(target, source, env): g.close() +def convert_size(size_bytes: int) -> str: + import math + + if size_bytes == 0: + return "0 bytes" + SIZE_NAMES = ["bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"] + index = math.floor(math.log(size_bytes, 1024)) + power = math.pow(1024, index) + size = round(size_bytes / power, 2) + return f"{size} {SIZE_NAMES[index]}" + + +def get_size(start_path: str = ".") -> int: + total_size = 0 + for dirpath, _, filenames in os.walk(start_path): + for file in filenames: + path = os.path.join(dirpath, file) + total_size += os.path.getsize(path) + return total_size + + +def is_binary(path: str) -> bool: + try: + with open(path, encoding="utf-8") as out: + out.read(1024) + return False + except UnicodeDecodeError: + return True + + +def clean_cache(cache_path: str, cache_limit: int, verbose: bool): + from glob import glob + + files = glob(os.path.join(cache_path, "*", "*")) + if not files: + return + + # Remove all text files, store binary files in list of (filename, size, atime). + purge = [] + texts = [] + stats = [] + for file in files: + try: + # Save file stats to rewrite after modifying. + tmp_stat = os.stat(file) + if is_binary(file): + stats.append((file, *tmp_stat[6:8])) + # Restore file stats after reading. + os.utime(file, (tmp_stat[7], tmp_stat[8])) + else: + texts.append(file) + except OSError: + print(f'Failed to access cache file "{file}"; skipping.') + + if texts: + count = len(texts) + for file in texts: + try: + os.remove(file) + except OSError: + print(f'Failed to remove cache file "{file}"; skipping.') + count -= 1 + if verbose: + print("Purging %d text %s from cache..." % (count, "files" if count > 1 else "file")) + + if cache_limit: + # Sort by most recent access (most sensible to keep) first. Search for the first entry where + # the cache limit is reached. + stats.sort(key=lambda x: x[2], reverse=True) + sum = 0 + for index, stat in enumerate(stats): + sum += stat[1] + if sum > cache_limit: + purge.extend([x[0] for x in stats[index:]]) + break + + if purge: + count = len(purge) + for file in purge: + try: + os.remove(file) + except OSError: + print(f'Failed to remove cache file "{file}"; skipping.') + count -= 1 + if verbose: + print("Purging %d %s from cache..." % (count, "files" if count > 1 else "file")) + + +def prepare_cache(env) -> None: + import atexit + + if env.GetOption("clean"): + return + + cache_path = None + if env["cache_path"]: + cache_path = env["cache_path"] + elif os.environ.get("SCONS_CACHE"): + print("Environment variable `SCONS_CACHE` is deprecated; use `cache_path` argument instead.") + cache_path = os.environ.get("SCONS_CACHE") + + if not cache_path: + return + + env.CacheDir(cache_path) + cache_path = env.get_CacheDir().path + print(f'SCons cache enabled... (path: "{cache_path}")') + + if env["cache_limit"]: + cache_limit = float(env["cache_limit"]) + elif os.environ.get("SCONS_CACHE_LIMIT"): + print("Environment variable `SCONS_CACHE_LIMIT` is deprecated; use `cache_limit` argument instead.") + cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", "0")) / 1024 # Old method used MiB, convert to GiB + + # Convert GiB to bytes; treat negative numbers as 0 (unlimited). + cache_limit = max(0, int(cache_limit * 1024 * 1024 * 1024)) + if env["verbose"]: + print( + "Current cache limit is {} (used: {})".format( + # FIXME: Infinity symbol `∞` breaks Windows GHA. + convert_size(cache_limit) if cache_limit else "", + convert_size(get_size(cache_path)), + ) + ) + + atexit.register(clean_cache, cache_path, cache_limit, env["verbose"]) + + def generate(env): + # Setup caching logic early to catch everything. + prepare_cache(env) + + # Renamed to `content-timestamp` in SCons >= 4.2, keeping MD5 for compat. + env.Decider("MD5-timestamp") + # Default num_jobs to local cpu count if not user specified. # SCons has a peculiarity where user-specified options won't be overridden # by SetOption, so we can rely on this to know if we should use our default.