From 57d16824166353af25404f8bee9f2bc65f00e633 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 19 Oct 2025 14:43:40 +0200 Subject: [PATCH 1/2] add a cachedir file to various directories that shouldnt be backed up --- src/Artifacts.jl | 4 +++- src/Operations.jl | 6 +++++- src/Pkg.jl | 11 +++++++++++ src/Registry/Registry.jl | 5 ++++- src/Types.jl | 5 +++-- test/new.jl | 5 +++++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 4833428165..11ac99c129 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -6,7 +6,7 @@ using Tar: can_symlink using FileWatching: FileWatching import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint, - ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write + ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write, ..create_cachedir_tag import Base: get, SHA1 import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, artifact_paths, @@ -31,6 +31,7 @@ function create_artifact(f::Function) # Ensure the `artifacts` directory exists in our default depot artifacts_dir = first(artifacts_dirs()) mkpath(artifacts_dir) + create_cachedir_tag(artifacts_dir) # Temporary directory where we'll do our creation business temp_dir = mktempdir(artifacts_dir) @@ -346,6 +347,7 @@ function download_artifact( # Ensure the `artifacts` directory exists in our default depot artifacts_dir = first(artifacts_dirs()) mkpath(artifacts_dir) + create_cachedir_tag(artifacts_dir) # expected artifact path dst = joinpath(artifacts_dir, bytes2hex(tree_hash.bytes)) diff --git a/src/Operations.jl b/src/Operations.jl index 591e0305e4..bf67a0ece3 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -16,7 +16,7 @@ using Base.BinaryPlatforms import ...Pkg import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE import ...Pkg: UPDATED_REGISTRY_THIS_SESSION, RESPECT_SYSIMAGE_VERSIONS, should_autoprecompile -import ...Pkg: usable_io, discover_repo +import ...Pkg: usable_io, discover_repo, create_cachedir_tag ######### # Utils # @@ -805,6 +805,7 @@ function install_archive( # files are on a different fs. So use a temp dir in the same depot dir as some systems might # be serving different parts of the depot on different filesystems via links i.e. pkgeval does this. depot_temp = mkpath(joinpath(dirname(dirname(version_path)), "temp")) # .julia/packages/temp + create_cachedir_tag(dirname(dirname(version_path))) tmp_objects = String[] url_success = false @@ -886,6 +887,7 @@ function install_git( try clones_dir = joinpath(depots1(), "clones") ispath(clones_dir) || mkpath(clones_dir) + create_cachedir_tag(clones_dir) repo_path = joinpath(clones_dir, string(uuid)) first_url = first(urls) repo = GitTools.ensure_clone( @@ -912,6 +914,7 @@ function install_git( tree isa LibGit2.GitTree || error("$name: git object $(string(hash)) should be a tree, not $(typeof(tree))") mkpath(version_path) + create_cachedir_tag(dirname(dirname(version_path))) GitTools.checkout_tree_to_path(repo, tree, version_path) return finally @@ -1541,6 +1544,7 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose = false, allow_r key = string(entry.tree_hash) scratch = joinpath(pkg_scratchpath(), key) mkpath(scratch) + create_cachedir_tag(joinpath(depots1(), "scratchspaces")) log_file = joinpath(scratch, "build.log") # Associate the logfile with the package being built dict = Dict{String, Any}( diff --git a/src/Pkg.jl b/src/Pkg.jl index ab9f84f7cd..ccbcfdc226 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -46,6 +46,17 @@ end logdir(depot = depots1()) = joinpath(depot, "logs") devdir(depot = depots1()) = get(ENV, "JULIA_PKG_DEVDIR", joinpath(depot, "dev")) envdir(depot = depots1()) = joinpath(depot, "environments") + +function create_cachedir_tag(cache_dir::AbstractString) + return try + tag_file = joinpath(cache_dir, "CACHEDIR.TAG") + if !isfile(tag_file) + write(tag_file, "Signature: 8a477f597d28d172789f06886806bc55\n# This file is a cache directory tag created by Julia Pkg.\n# See https://bford.info/cachedir/\n") + end + catch + # Ignore errors to avoid failing operations on read-only filesystems + end +end const UPDATED_REGISTRY_THIS_SESSION = Ref(false) const OFFLINE_MODE = Ref(false) const RESPECT_SYSIMAGE_VERSIONS = Ref(true) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index dba6c3a88f..d5b984a486 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -2,7 +2,7 @@ module Registry import ..Pkg using ..Pkg: depots, depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server, - GitTools, atomic_toml_write + GitTools, atomic_toml_write, create_cachedir_tag using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z, verify_archive_tree_hash using UUIDs, LibGit2, TOML, Dates import FileWatching @@ -209,6 +209,7 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S registry_update_log = get_registry_update_log() regdir = joinpath(target_depot, "registries") isdir(regdir) || mkpath(regdir) + create_cachedir_tag(regdir) # only allow one julia process to download and install registries at a time FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do # once we're pidlocked check if another process has installed any of the registries @@ -421,6 +422,7 @@ end function save_registry_update_log(d::Dict) pkg_scratch_space = joinpath(DEPOT_PATH[1], "scratchspaces", "44cfe95a-1eb2-52ea-b672-e2afdf69b78f") mkpath(pkg_scratch_space) + create_cachedir_tag(joinpath(DEPOT_PATH[1], "scratchspaces")) pkg_reg_updated_file = joinpath(pkg_scratch_space, "registry_updates.toml") return atomic_toml_write(pkg_reg_updated_file, d) end @@ -455,6 +457,7 @@ function update(regs::Vector{RegistrySpec}; io::IO = stderr_f(), force::Bool = t depot_regs = isempty(regs) ? reachable_registries(; depots = depot) : regs regdir = joinpath(depot, "registries") isdir(regdir) || mkpath(regdir) + create_cachedir_tag(regdir) # only allow one julia process to update registries in this depot at a time FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do errors = Tuple{String, String}[] diff --git a/src/Types.jl b/src/Types.jl index 15594dab2a..309244d538 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -10,7 +10,7 @@ import Base.string using TOML import ..Pkg, ..Registry -import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS, atomic_toml_write +import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS, atomic_toml_write, create_cachedir_tag import Base.BinaryPlatforms: Platform using ..Pkg.Versions import FileWatching @@ -944,7 +944,7 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec) LibGit2.with(GitTools.ensure_clone(ctx.io, add_repo_cache_path(repo_source::Union{Nothing, String}), repo_source::Union{Nothing, String}; isbare = true)) do repo repo_source_typed = repo_source::Union{Nothing, String} GitTools.check_valid_HEAD(repo) - + create_cachedir_tag(dirname(add_repo_cache_path(repo_source))) # If the user didn't specify rev, assume they want the default (master) branch if on a branch, otherwise the current commit if pkg.repo.rev === nothing pkg.repo.rev = LibGit2.isattached(repo) ? LibGit2.branch(repo) : string(LibGit2.GitHash(LibGit2.head(repo))) @@ -1009,6 +1009,7 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec) # Otherwise, move the temporary path into its correct place and set read only mkpath(version_path) mv(temp_path, version_path; force = true) + create_cachedir_tag(dirname(dirname(version_path))) set_readonly(version_path) return true end diff --git a/test/new.jl b/test/new.jl index 2ebe0fd46e..ed50726e78 100644 --- a/test/new.jl +++ b/test/new.jl @@ -46,6 +46,9 @@ Pkg._auto_gc_enabled[] = false reg = regs[1] @test reg.name == "General" @test reg.uuid == general_uuid + # - Check that CACHEDIR.TAG files exist in cache directories + @test isfile(joinpath(LOADED_DEPOT, "registries", "CACHEDIR.TAG")) + @test isfile(joinpath(LOADED_DEPOT, "packages", "CACHEDIR.TAG")) # - The package should be installed correctly. source053, source053_time = nothing, nothing Pkg.dependencies(exuuid) do pkg @@ -90,6 +93,7 @@ Pkg._auto_gc_enabled[] = false end # Now check packages which track repos instead of registered versions Pkg.add(url = "https://github.com/JuliaLang/Example.jl", rev = "v0.5.3") + @test isfile(joinpath(LOADED_DEPOT, "clones", "CACHEDIR.TAG")) Pkg.dependencies(exuuid) do pkg @test !pkg.is_tracking_registry @test isdir(pkg.source) @@ -2728,6 +2732,7 @@ end "44cfe95a-1eb2-52ea-b672-e2afdf69b78f", "f99d57aad0e5eb2434491b47bac92bb88d463001", "build.log" ) @test isfile(log_file_add) + @test isfile(joinpath(DEPOT_PATH[1], "scratchspaces", "CACHEDIR.TAG")) @test occursin("oops", read(log_file_add, String)) end end From 49afc53b7b1cc832aa51fff2199848eccde80e0a Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 20 Oct 2025 13:50:14 +0200 Subject: [PATCH 2/2] relax some isempty tests --- test/pkg.jl | 4 ++-- test/registry.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 07fe417a30..23380d138d 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -189,7 +189,7 @@ temp_pkg_dir() do project_path # Test that unused packages are reaped Pkg.gc() - @test isempty(readdir(pkgdir)) + @test isempty(filter(x -> x != "CACHEDIR.TAG", readdir(pkgdir))) clonedir = joinpath(Pkg.depots1(), "clones") Pkg.add(Pkg.PackageSpec(name = TEST_PKG.name, rev = "master")) @@ -197,7 +197,7 @@ temp_pkg_dir() do project_path Pkg.rm(TEST_PKG.name) # Test that unused repos are also reaped Pkg.gc() - @test isempty(readdir(clonedir)) + @test isempty(filter(x -> x != "CACHEDIR.TAG", readdir(clonedir))) end @testset "package with wrong UUID" begin diff --git a/test/registry.jl b/test/registry.jl index ad30823bd9..f89219d4ef 100644 --- a/test/registry.jl +++ b/test/registry.jl @@ -443,7 +443,7 @@ if Pkg.Registry.registry_use_pkg_server() end Pkg.update() Pkg.Registry.rm(name = "General") - @test isempty(readdir(joinpath(DEPOT_PATH[1], "registries"))) + @test isempty(filter(x -> x != "CACHEDIR.TAG", readdir(joinpath(DEPOT_PATH[1], "registries")))) end end end