use shallow clones for registry/packages (when supported by LibGit2) (#4487)

Author: KristofferC

src/API.jl

@@ -1346,11 +1346,11 @@ function instantiate(
         end
         repo_path = Types.add_repo_cache_path(repo_source)
         let repo_source = repo_source
-            LibGit2.with(GitTools.ensure_clone(ctx.io, repo_path, repo_source; isbare = true)) do repo
+            LibGit2.with(GitTools.ensure_clone(ctx.io, repo_path, repo_source; isbare = true, depth = 1)) do repo
                 # We only update the clone if the tree hash can't be found
                 tree_hash_object = tree_hash(repo, string(pkg.tree_hash))
                 if tree_hash_object === nothing
-                    GitTools.fetch(ctx.io, repo, repo_source; refspecs = Types.refspecs)
+                    GitTools.fetch(ctx.io, repo, repo_source; refspecs = Types.refspecs, depth = LibGit2.Consts.FETCH_DEPTH_UNSHALLOW)
                     tree_hash_object = tree_hash(repo, string(pkg.tree_hash))
                 end
                 if tree_hash_object === nothing

src/GitTools.jl

@@ -13,6 +13,45 @@ using Printf
 use_cli_git() = Base.get_bool_env("JULIA_PKG_USE_CLI_GIT", false)
 const RESOLVING_DELTAS_HEADER = "Resolving Deltas:"
 
+# Check if LibGit2 supports shallow clones (requires LibGit2 >= 1.7.0)
+# We check both the LibGit2 version and the existence of `isshallow` to ensure
+# the shallow clone functionality is available
+function supports_shallow_clone()
+    # This seems buggy on Windows? Get some weird CI errors with it.
+    if Sys.iswindows()
+        return false
+    end
+    has_version = @static if isdefined(LibGit2, :VERSION)
+        LibGit2.VERSION >= v"1.7.0"
+    else
+        false
+    end
+    has_isshallow = isdefined(LibGit2, :isshallow)
+    return has_version && has_isshallow
+end
+
+# Check if a URL is a local path or file:// URL
+# Shallow clones are only supported for network protocols (HTTP, HTTPS, Git, SSH)
+function is_local_repo(url::AbstractString)
+    # Check if it's a local filesystem path
+    ispath(url) && return true
+    # Check if it uses file:// protocol
+    startswith(url, "file://") && return true
+    return false
+end
+
+# Check if a repository is a shallow clone
+function isshallow(repo::LibGit2.GitRepo)
+    if supports_shallow_clone() && isdefined(LibGit2, :isshallow)
+        return LibGit2.isshallow(repo)
+    else
+        # Fallback: check for .git/shallow file
+        repo_path = LibGit2.path(repo)
+        shallow_file = joinpath(repo_path, "shallow")
+        return isfile(shallow_file)
+    end
+end
+
 function transfer_progress(progress::Ptr{LibGit2.TransferProgress}, p::Any)
     progress = unsafe_load(progress)
     @assert haskey(p, :transfer_progress)
@@ -89,11 +128,17 @@ function checkout_tree_to_path(repo::LibGit2.GitRepo, tree::LibGit2.GitObject, p
     end
 end
 
-function clone(io::IO, url, source_path; header = nothing, credentials = nothing, isbare = false, kwargs...)
+function clone(io::IO, url, source_path; header = nothing, credentials = nothing, isbare = false, depth::Integer = 0, kwargs...)
     url = String(url)::String
     source_path = String(source_path)::String
     @assert !isdir(source_path) || isempty(readdir(source_path))
     url = normalize_url(url)
+
+    # Disable shallow clones for local repos (not supported) or if LibGit2 doesn't support it
+    if depth > 0 && (is_local_repo(url) || !supports_shallow_clone())
+        depth = 0
+    end
+
     printpkgstyle(io, :Cloning, header === nothing ? "git-repo `$url`" : header)
     bar = MiniProgressBar(header = "Cloning:", color = Base.info_color())
     fancyprint = can_fancyprint(io)
@@ -103,8 +148,10 @@ function clone(io::IO, url, source_path; header = nothing, credentials = nothing
     end
     return try
         if use_cli_git()
-            args = ["--quiet", url, source_path]
-            isbare && pushfirst!(args, "--bare")
+            args = ["--quiet"]
+            depth > 0 && push!(args, "--depth=$depth")
+            isbare && push!(args, "--bare")
+            push!(args, url, source_path)
             cmd = `git clone $args`
             try
                 run(pipeline(cmd; stdout = devnull))
@@ -124,7 +171,12 @@ function clone(io::IO, url, source_path; header = nothing, credentials = nothing
                 LibGit2.Callbacks()
             end
             mkpath(source_path)
-            return LibGit2.clone(url, source_path; callbacks, credentials, isbare, kwargs...)
+            # Only pass depth if shallow clones are supported and depth > 0
+            if depth > 0
+                return LibGit2.clone(url, source_path; callbacks, credentials, isbare, depth, kwargs...)
+            else
+                return LibGit2.clone(url, source_path; callbacks, credentials, isbare, kwargs...)
+            end
         end
     catch err
         rm(source_path; force = true, recursive = true)
@@ -149,10 +201,16 @@ function geturl(repo)
     end
 end
 
-function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl = nothing; header = nothing, credentials = nothing, refspecs = [""], kwargs...)
+function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl = nothing; header = nothing, credentials = nothing, refspecs = [""], depth::Integer = 0, kwargs...)
     if remoteurl === nothing
         remoteurl = geturl(repo)
     end
+
+    # Disable shallow fetches for local repos (not supported) or if LibGit2 doesn't support it
+    if depth > 0 && (is_local_repo(remoteurl) || !supports_shallow_clone())
+        depth = 0
+    end
+
     fancyprint = can_fancyprint(io)
     remoteurl = normalize_url(remoteurl)
     printpkgstyle(io, :Updating, header === nothing ? "git-repo `$remoteurl`" : header)
@@ -174,15 +232,23 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl = nothing; header = noth
     return try
         if use_cli_git()
             let remoteurl = remoteurl
-                cmd = `git -C $(LibGit2.path(repo)) fetch -q $remoteurl $(only(refspecs))`
+                args = ["-C", LibGit2.path(repo), "fetch", "-q"]
+                depth > 0 && push!(args, "--depth=$depth")
+                push!(args, remoteurl, only(refspecs))
+                cmd = `git $args`
                 try
                     run(pipeline(cmd; stdout = devnull))
                 catch err
                     Pkg.Types.pkgerror("The command $(cmd) failed, error: $err")
                 end
             end
         else
-            return LibGit2.fetch(repo; remoteurl, callbacks, credentials, refspecs, kwargs...)
+            # Only pass depth if shallow clones are supported and depth > 0
+            if depth > 0
+                return LibGit2.fetch(repo; remoteurl, callbacks, credentials, refspecs, depth, kwargs...)
+            else
+                return LibGit2.fetch(repo; remoteurl, callbacks, credentials, refspecs, kwargs...)
+            end
         end
     catch err
         err isa LibGit2.GitError || rethrow()

src/Operations.jl

@@ -1154,7 +1154,7 @@ function install_git(
         first_url = first(urls)
         repo = GitTools.ensure_clone(
             io, repo_path, first_url; isbare = true,
-            header = "[$uuid] $name from $first_url"
+            header = "[$uuid] $name from $first_url", depth = 1
         )
         git_hash = LibGit2.GitHash(hash.bytes)
         for url in urls
@@ -1165,7 +1165,7 @@ function install_git(
             catch err
                 err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow()
             end
-            GitTools.fetch(io, repo, url, refspecs = refspecs)
+            GitTools.fetch(io, repo, url, refspecs = refspecs, depth = LibGit2.Consts.FETCH_DEPTH_UNSHALLOW)
         end
         tree = try
             LibGit2.GitObject(repo, git_hash)

src/Registry/Registry.jl

@@ -351,7 +351,8 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{S
                         return
                     elseif reg.url !== nothing # clone from url
                         # retry to help spurious connection issues, particularly on CI
-                        repo = retry(GitTools.clone, delays = fill(1.0, 5), check = (s, e) -> isa(e, LibGit2.GitError))(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
+                        # Use shallow clone (depth=1) for registries since we only need the latest state
+                        repo = retry(GitTools.clone, delays = fill(1.0, 5), check = (s, e) -> isa(e, LibGit2.GitError))(io, reg.url, tmp; header = "registry from $(repr(reg.url))", depth = 1)
                         LibGit2.close(repo)
                     else
                         Pkg.Types.pkgerror("no path or url specified for registry")
@@ -632,7 +633,9 @@ function update(regs::Vector{RegistrySpec}; io::IO = stderr_f(), force::Bool = t
                                 end
                                 branch = LibGit2.headname(repo)
                                 try
-                                    GitTools.fetch(io, repo; refspecs = ["+refs/heads/$branch:refs/remotes/origin/$branch"])
+                                    # If this is a shallow clone, continue using shallow fetches
+                                    fetch_depth = GitTools.isshallow(repo) ? 1 : 0
+                                    GitTools.fetch(io, repo; refspecs = ["+refs/heads/$branch:refs/remotes/origin/$branch"], depth = fetch_depth)
                                 catch e
                                     e isa Pkg.Types.PkgError || rethrow()
                                     push!(errors, (reg.path, "failed to fetch from repo: $(e.msg)"))

src/Types.jl

@@ -1010,7 +1010,7 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec)
 
     return let repo_source = repo_source
         # The type-assertions below are necessary presumably due to julia#36454
-        LibGit2.with(GitTools.ensure_clone(ctx.io, add_repo_cache_path(repo_source::Union{Nothing, String}), repo_source::Union{Nothing, String}; isbare = true)) do repo
+        LibGit2.with(GitTools.ensure_clone(ctx.io, add_repo_cache_path(repo_source::Union{Nothing, String}), repo_source::Union{Nothing, String}; isbare = true, depth = 1)) do repo
             repo_source_typed = repo_source::Union{Nothing, String}
             GitTools.check_valid_HEAD(repo)
             create_cachedir_tag(dirname(add_repo_cache_path(repo_source)))
@@ -1027,11 +1027,11 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec)
                 if startswith(rev_or_hash, "pull/") && endswith(rev_or_hash, "/head")
                     pr_number = rev_or_hash[6:(end - 5)]  # Extract number from "pull/X/head"
                     pr_refspecs = ["+refs/pull/$(pr_number)/head:refs/cache/pull/$(pr_number)/head"]
-                    GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = pr_refspecs)
+                    GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = pr_refspecs, depth = 1)
                     # For branch names, fetch only the specific branch
                 elseif !looks_like_commit_hash(string(rev_or_hash))
                     specific_refspec = ["+refs/heads/$(rev_or_hash):refs/cache/heads/$(rev_or_hash)"]
-                    GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = specific_refspec)
+                    GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = specific_refspec, depth = 1)
                 else
                     # For commit hashes, fetch all branches
                     GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = refspecs)
@@ -1054,7 +1054,7 @@ function handle_repo_add!(ctx::Context, pkg::PackageSpec)
             if isbranch && !fetched && !ispinned
                 # Fetch only the specific branch being tracked
                 specific_refspec = ["+refs/heads/$(rev_or_hash):refs/cache/heads/$(rev_or_hash)"]
-                GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = specific_refspec)
+                GitTools.fetch(ctx.io, repo, repo_source_typed; refspecs = specific_refspec, depth = 1)
                 gitobject, isbranch = get_object_or_branch(repo, rev_or_hash)
             end