add a cachedir file to various directories that shouldnt be backed up (#4438)

Author: KristofferC

src/Artifacts.jl

@@ -6,7 +6,7 @@ using Tar: can_symlink
 using FileWatching: FileWatching
 
 import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint,
-    ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write
+    ..stderr_f, ..printpkgstyle, ..mv_temp_dir_retries, ..atomic_toml_write, ..create_cachedir_tag
 
 import Base: get, SHA1
 import Artifacts: artifact_names, ARTIFACTS_DIR_OVERRIDE, ARTIFACT_OVERRIDES, artifact_paths,
@@ -31,6 +31,7 @@ function create_artifact(f::Function)
     # Ensure the `artifacts` directory exists in our default depot
     artifacts_dir = first(artifacts_dirs())
     mkpath(artifacts_dir)
+    create_cachedir_tag(artifacts_dir)
 
     # Temporary directory where we'll do our creation business
     temp_dir = mktempdir(artifacts_dir)
@@ -346,6 +347,7 @@ function download_artifact(
         # Ensure the `artifacts` directory exists in our default depot
         artifacts_dir = first(artifacts_dirs())
         mkpath(artifacts_dir)
+        create_cachedir_tag(artifacts_dir)
         # expected artifact path
         dst = joinpath(artifacts_dir, bytes2hex(tree_hash.bytes))
 

src/Operations.jl

@@ -16,7 +16,7 @@ using Base.BinaryPlatforms
 import ...Pkg
 import ...Pkg: pkg_server, Registry, pathrepr, can_fancyprint, printpkgstyle, stderr_f, OFFLINE_MODE
 import ...Pkg: UPDATED_REGISTRY_THIS_SESSION, RESPECT_SYSIMAGE_VERSIONS, should_autoprecompile
-import ...Pkg: usable_io, discover_repo
+import ...Pkg: usable_io, discover_repo, create_cachedir_tag
 
 #########
 # Utils #
@@ -805,6 +805,7 @@ function install_archive(
     # files are on a different fs. So use a temp dir in the same depot dir as some systems might
     # be serving different parts of the depot on different filesystems via links i.e. pkgeval does this.
     depot_temp = mkpath(joinpath(dirname(dirname(version_path)), "temp")) # .julia/packages/temp
+    create_cachedir_tag(dirname(dirname(version_path)))
 
     tmp_objects = String[]
     url_success = false
@@ -886,6 +887,7 @@ function install_git(
     try
         clones_dir = joinpath(depots1(), "clones")
         ispath(clones_dir) || mkpath(clones_dir)
+        create_cachedir_tag(clones_dir)
         repo_path = joinpath(clones_dir, string(uuid))
         first_url = first(urls)
         repo = GitTools.ensure_clone(
@@ -912,6 +914,7 @@ function install_git(
         tree isa LibGit2.GitTree ||
             error("$name: git object $(string(hash)) should be a tree, not $(typeof(tree))")
         mkpath(version_path)
+        create_cachedir_tag(dirname(dirname(version_path)))
         GitTools.checkout_tree_to_path(repo, tree, version_path)
         return
     finally
@@ -1541,6 +1544,7 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose = false, allow_r
                 key = string(entry.tree_hash)
                 scratch = joinpath(pkg_scratchpath(), key)
                 mkpath(scratch)
+                create_cachedir_tag(joinpath(depots1(), "scratchspaces"))
                 log_file = joinpath(scratch, "build.log")
                 # Associate the logfile with the package being built
                 dict = Dict{String, Any}(

src/Pkg.jl

@@ -46,6 +46,17 @@ end
 logdir(depot = depots1()) = joinpath(depot, "logs")
 devdir(depot = depots1()) = get(ENV, "JULIA_PKG_DEVDIR", joinpath(depot, "dev"))
 envdir(depot = depots1()) = joinpath(depot, "environments")
+
+function create_cachedir_tag(cache_dir::AbstractString)
+    return try
+        tag_file = joinpath(cache_dir, "CACHEDIR.TAG")
+        if !isfile(tag_file)
+            write(tag_file, "Signature: 8a477f597d28d172789f06886806bc55\n# This file is a cache directory tag created by Julia Pkg.\n# See https://bford.info/cachedir/\n")
+        end
+    catch
+        # Ignore errors to avoid failing operations on read-only filesystems
+    end
+end
 const UPDATED_REGISTRY_THIS_SESSION = Ref(false)
 const OFFLINE_MODE = Ref(false)
 const RESPECT_SYSIMAGE_VERSIONS = Ref(true)

src/Registry/Registry.jl

@@ -40,7 +40,7 @@ module Registry
 
 import ..Pkg
 using ..Pkg: depots, depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server,
-    GitTools, atomic_toml_write
+    GitTools, atomic_toml_write, create_cachedir_tag
 using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z, verify_archive_tree_hash
 using UUIDs, LibGit2, TOML, Dates
 import FileWatching
@@ -247,6 +247,7 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S
     registry_update_log = get_registry_update_log()
     regdir = joinpath(target_depot, "registries")
     isdir(regdir) || mkpath(regdir)
+    create_cachedir_tag(regdir)
     # only allow one julia process to download and install registries at a time
     FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do
         # once we're pidlocked check if another process has installed any of the registries
@@ -459,6 +460,7 @@ end
 function save_registry_update_log(d::Dict)
     pkg_scratch_space = joinpath(DEPOT_PATH[1], "scratchspaces", "44cfe95a-1eb2-52ea-b672-e2afdf69b78f")
     mkpath(pkg_scratch_space)
+    create_cachedir_tag(joinpath(DEPOT_PATH[1], "scratchspaces"))
     pkg_reg_updated_file = joinpath(pkg_scratch_space, "registry_updates.toml")
     return atomic_toml_write(pkg_reg_updated_file, d)
 end
@@ -493,6 +495,7 @@ function update(regs::Vector{RegistrySpec}; io::IO = stderr_f(), force::Bool = t
         depot_regs = isempty(regs) ? reachable_registries(; depots = depot) : regs
         regdir = joinpath(depot, "registries")
         isdir(regdir) || mkpath(regdir)
+        create_cachedir_tag(regdir)
         # only allow one julia process to update registries in this depot at a time
         FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do
             errors = Tuple{String, String}[]

src/Types.jl

@@ -10,7 +10,7 @@ import Base.string
 
 using TOML
 import ..Pkg, ..Registry
-import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS, atomic_toml_write
+import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS, atomic_toml_write, create_cachedir_tag
 import Base.BinaryPlatforms: Platform
 using ..Pkg.Versions
 import FileWatching
@@ -955,7 +955,7 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec)
         LibGit2.with(GitTools.ensure_clone(ctx.io, add_repo_cache_path(repo_source::Union{Nothing, String}), repo_source::Union{Nothing, String}; isbare = true)) do repo
             repo_source_typed = repo_source::Union{Nothing, String}
             GitTools.check_valid_HEAD(repo)
-
+            create_cachedir_tag(dirname(add_repo_cache_path(repo_source)))
             # If the user didn't specify rev, assume they want the default (master) branch if on a branch, otherwise the current commit
             if pkg.repo.rev === nothing
                 pkg.repo.rev = LibGit2.isattached(repo) ? LibGit2.branch(repo) : string(LibGit2.GitHash(LibGit2.head(repo)))
@@ -1020,6 +1020,7 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec)
             # Otherwise, move the temporary path into its correct place and set read only
             mkpath(version_path)
             mv(temp_path, version_path; force = true)
+            create_cachedir_tag(dirname(dirname(version_path)))
             set_readonly(version_path)
             return true
         end

test/new.jl

@@ -46,6 +46,9 @@ Pkg._auto_gc_enabled[] = false
         reg = regs[1]
         @test reg.name == "General"
         @test reg.uuid == general_uuid
+        # - Check that CACHEDIR.TAG files exist in cache directories
+        @test isfile(joinpath(LOADED_DEPOT, "registries", "CACHEDIR.TAG"))
+        @test isfile(joinpath(LOADED_DEPOT, "packages", "CACHEDIR.TAG"))
         # - The package should be installed correctly.
         source053, source053_time = nothing, nothing
         Pkg.dependencies(exuuid) do pkg
@@ -90,6 +93,7 @@ Pkg._auto_gc_enabled[] = false
         end
         # Now check packages which track repos instead of registered versions
         Pkg.add(url = "https://github.com/JuliaLang/Example.jl", rev = "v0.5.3")
+        @test isfile(joinpath(LOADED_DEPOT, "clones", "CACHEDIR.TAG"))
         Pkg.dependencies(exuuid) do pkg
             @test !pkg.is_tracking_registry
             @test isdir(pkg.source)
@@ -2728,6 +2732,7 @@ end
                 "44cfe95a-1eb2-52ea-b672-e2afdf69b78f", "f99d57aad0e5eb2434491b47bac92bb88d463001", "build.log"
             )
             @test isfile(log_file_add)
+            @test isfile(joinpath(DEPOT_PATH[1], "scratchspaces", "CACHEDIR.TAG"))
             @test occursin("oops", read(log_file_add, String))
         end
     end

test/pkg.jl

@@ -189,15 +189,15 @@ temp_pkg_dir() do project_path
 
         # Test that unused packages are reaped
         Pkg.gc()
-        @test isempty(readdir(pkgdir))
+        @test isempty(filter(x -> x != "CACHEDIR.TAG", readdir(pkgdir)))
 
         clonedir = joinpath(Pkg.depots1(), "clones")
         Pkg.add(Pkg.PackageSpec(name = TEST_PKG.name, rev = "master"))
         @test !isempty(readdir(clonedir))
         Pkg.rm(TEST_PKG.name)
         # Test that unused repos are also reaped
         Pkg.gc()
-        @test isempty(readdir(clonedir))
+        @test isempty(filter(x -> x != "CACHEDIR.TAG", readdir(clonedir)))
     end
 
     @testset "package with wrong UUID" begin

test/registry.jl

@@ -443,7 +443,7 @@ if Pkg.Registry.registry_use_pkg_server()
                     end
                     Pkg.update()
                     Pkg.Registry.rm(name = "General")
-                    @test isempty(readdir(joinpath(DEPOT_PATH[1], "registries")))
+                    @test isempty(filter(x -> x != "CACHEDIR.TAG", readdir(joinpath(DEPOT_PATH[1], "registries"))))
                 end
             end
         end