From 317931ea8661145b848359c72a8eb1a4a2106b03 Mon Sep 17 00:00:00 2001 From: cgarling Date: Sat, 12 Jul 2025 19:59:50 -0400 Subject: [PATCH 01/11] Support git-lfs (large file storage) --- Project.toml | 4 +++- src/git_function.jl | 8 +++++++- test/runtests.jl | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 2619f85..4c8710a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,14 +1,16 @@ name = "Git" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" authors = ["Dilum Aluthge", "contributors"] +version = "1.5.0" [deps] +Git_LFS_jll = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" Git_jll = "f8c6e375-362e-5223-8a59-34ff63f689eb" JLLWrappers = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" OpenSSH_jll = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" [compat] +Git_LFS_jll = "3.7" Git_jll = "2.44" JLLWrappers = "1.1" OpenSSH_jll = "9, 10" diff --git a/src/git_function.jl b/src/git_function.jl index 1e34733..6ee5a6a 100644 --- a/src/git_function.jl +++ b/src/git_function.jl @@ -1,4 +1,5 @@ using OpenSSH_jll: OpenSSH_jll +using Git_LFS_jll: Git_LFS_jll using JLLWrappers: pathsep, LIBPATH_env """ @@ -52,8 +53,8 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) end # Use OpenSSH from the JLL: . + path = split(get(ENV, "PATH", ""), pathsep) if !Sys.iswindows() && OpenSSH_jll.is_available() - path = split(get(ENV, "PATH", ""), pathsep) libpath = split(get(ENV, LIBPATH_env, ""), pathsep) path = vcat(dirname(OpenSSH_jll.ssh_path), path) @@ -67,6 +68,11 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep), LIBPATH_env => join(libpath, pathsep)) end + # Add git-lfs + if Git_LFS_jll.is_available() + path = vcat(dirname(Git_LFS_jll.git_lfs_path), path) + git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep)) + end return git_cmd end diff --git a/test/runtests.jl b/test/runtests.jl index 6c539a2..3603799 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,6 +34,20 @@ end @test isdir("Git.jl") @test isfile(joinpath("Git.jl", "Project.toml")) end + + # git-lfs tests + withtempdir() do tmp_dir + rname = "repo-with-large-file-storage" + @test !isdir(rname) + @test !isfile(joinpath(rname, "LargeFile.zip")) + run(`$(git()) clone https://github.com/Apress/repo-with-large-file-storage`) + run(`$(git()) -C $rname lfs install --local`) + run(`$(git()) -C $rname lfs pull`) + @test isdir(rname) + @test isfile(joinpath(rname, "LargeFile.zip")) + # Test filesize to make sure we got real file and not small LFS pointer file + @test filesize(joinpath(rname, "LargeFile.zip")) > 10^6 + end end @testset "Safety" begin From 5bb20c1a821189fd3dff70432e09bd035b875502 Mon Sep 17 00:00:00 2001 From: cgarling Date: Sat, 12 Jul 2025 20:08:05 -0400 Subject: [PATCH 02/11] Update readme to state git LFS support --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 337a302..e9d4866 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ not need to have Git installed on your computer, and neither do the users of your packages! Git.jl provides a Git binary via -[Git_jll.jl](https://github.com/JuliaBinaryWrappers/Git_jll.jl). +[Git_jll.jl](https://github.com/JuliaBinaryWrappers/Git_jll.jl) +and a Git LFS binary for large file support via +[Git_LFS_jll.jl](https://github.com/JuliaBinaryWrappers/Git_LFS_jll.jl). The latest version of Git.jl requires at least Julia 1.6. Git.jl is intended to work on any platform that supports Julia, From 4d63629497b3569b569ef4d0ec03d14d05cd060e Mon Sep 17 00:00:00 2001 From: cgarling Date: Sat, 12 Jul 2025 20:17:14 -0400 Subject: [PATCH 03/11] Silence lfs git tests --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 3603799..b64b2c9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -40,9 +40,9 @@ end rname = "repo-with-large-file-storage" @test !isdir(rname) @test !isfile(joinpath(rname, "LargeFile.zip")) - run(`$(git()) clone https://github.com/Apress/repo-with-large-file-storage`) - run(`$(git()) -C $rname lfs install --local`) - run(`$(git()) -C $rname lfs pull`) + run(`$(git()) clone --quiet https://github.com/Apress/repo-with-large-file-storage`) + run(pipeline(`$(git()) -C $rname lfs install --local`; stdout=devnull)) + run(pipeline(`$(git()) -C $rname lfs pull`; stdout=devnull)) @test isdir(rname) @test isfile(joinpath(rname, "LargeFile.zip")) # Test filesize to make sure we got real file and not small LFS pointer file From b98ca06af35a8880b0a0c9b664748fd9048ede3b Mon Sep 17 00:00:00 2001 From: Chris Garling Date: Wed, 16 Jul 2025 10:12:06 -0400 Subject: [PATCH 04/11] Check for CI private key in test --- test/runtests.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index b64b2c9..561c269 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -90,7 +90,8 @@ end @testset "OpenSSH integration" begin is_ci = parse(Bool, strip(get(ENV, "CI", "false"))) is_gha = parse(Bool, strip(get(ENV, "GITHUB_ACTIONS", "false"))) - if is_ci && is_gha + has_privkey = "CI_READONLY_DEPLOYKEY_FOR_CI_TESTSUITE_PRIVATEKEY" ∈ keys(ENV) + if is_ci && is_gha && has_privkey @info "This is GitHub Actions CI, so running the OpenSSH test..." mktempdir() do sshprivkeydir privkey_filepath = joinpath(sshprivkeydir, "my_private_key") From bf81c752f58a644023e53d383cee4c727a4292c8 Mon Sep 17 00:00:00 2001 From: Chris Garling Date: Wed, 16 Jul 2025 10:18:37 -0400 Subject: [PATCH 05/11] fix check on CI private key --- test/runtests.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 561c269..471fa52 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -90,13 +90,12 @@ end @testset "OpenSSH integration" begin is_ci = parse(Bool, strip(get(ENV, "CI", "false"))) is_gha = parse(Bool, strip(get(ENV, "GITHUB_ACTIONS", "false"))) - has_privkey = "CI_READONLY_DEPLOYKEY_FOR_CI_TESTSUITE_PRIVATEKEY" ∈ keys(ENV) - if is_ci && is_gha && has_privkey + ssh_privkey = get(ENV, "CI_READONLY_DEPLOYKEY_FOR_CI_TESTSUITE_PRIVATEKEY", nothing) + if is_ci && is_gha && !isnothing(ssh_privkey) @info "This is GitHub Actions CI, so running the OpenSSH test..." mktempdir() do sshprivkeydir privkey_filepath = joinpath(sshprivkeydir, "my_private_key") open(privkey_filepath, "w") do io - ssh_privkey = ENV["CI_READONLY_DEPLOYKEY_FOR_CI_TESTSUITE_PRIVATEKEY"] println(io, ssh_privkey) end # open # We need to chmod our private key to 600, or SSH will ignore it. From d539c218ce309fac42b5fc1301d3170fc9f81156 Mon Sep 17 00:00:00 2001 From: Chris Garling Date: Wed, 16 Jul 2025 10:50:48 -0400 Subject: [PATCH 06/11] update env var check --- test/runtests.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 471fa52..75c61ea 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -88,11 +88,9 @@ end # https://github.com/JuliaVersionControl/Git.jl/issues/51 @testset "OpenSSH integration" begin - is_ci = parse(Bool, strip(get(ENV, "CI", "false"))) - is_gha = parse(Bool, strip(get(ENV, "GITHUB_ACTIONS", "false"))) - ssh_privkey = get(ENV, "CI_READONLY_DEPLOYKEY_FOR_CI_TESTSUITE_PRIVATEKEY", nothing) - if is_ci && is_gha && !isnothing(ssh_privkey) - @info "This is GitHub Actions CI, so running the OpenSSH test..." + ssh_privkey = get(ENV, "CI_READONLY_DEPLOYKEY_FOR_CI_TESTSUITE_PRIVATEKEY", "") + if !isempty(ssh_privkey) + @info "CI private key available, so running the OpenSSH test..." mktempdir() do sshprivkeydir privkey_filepath = joinpath(sshprivkeydir, "my_private_key") open(privkey_filepath, "w") do io From b9196af4a5c087e52da00bc301fe912473fec348 Mon Sep 17 00:00:00 2001 From: cgarling Date: Fri, 18 Jul 2025 11:10:42 -0400 Subject: [PATCH 07/11] Read path from `git_cmd`, not global The path can be modified by preceding code, so we need to read it from `git_cmd`, append the `git-lfs` directory, then write back to `git_cmd`. --- src/git_function.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/git_function.jl b/src/git_function.jl index 6ee5a6a..07f66cd 100644 --- a/src/git_function.jl +++ b/src/git_function.jl @@ -53,8 +53,8 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) end # Use OpenSSH from the JLL: . - path = split(get(ENV, "PATH", ""), pathsep) if !Sys.iswindows() && OpenSSH_jll.is_available() + path = split(get(ENV, "PATH", ""), pathsep) libpath = split(get(ENV, LIBPATH_env, ""), pathsep) path = vcat(dirname(OpenSSH_jll.ssh_path), path) @@ -70,6 +70,14 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # Add git-lfs if Git_LFS_jll.is_available() + # Read path from git_cmd.env as it can be modified above + idx = findfirst(startswith("PATH="), git_cmd.env) + path = if isnothing(idx) + "" + else + # dropping the `PATH=` part + git_cmd.env[idx][6:end] + end path = vcat(dirname(Git_LFS_jll.git_lfs_path), path) git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep)) end From fc31cd4c36d838c713fa3e024e92531e3d37975b Mon Sep 17 00:00:00 2001 From: cgarling Date: Fri, 18 Jul 2025 14:52:10 -0400 Subject: [PATCH 08/11] Try using `windows_verbatim` if git-lfs loaded CI issue could be related to single quotes in `cmd.exec` entries? --- src/git_function.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/git_function.jl b/src/git_function.jl index 07f66cd..e0e325e 100644 --- a/src/git_function.jl +++ b/src/git_function.jl @@ -79,6 +79,7 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) git_cmd.env[idx][6:end] end path = vcat(dirname(Git_LFS_jll.git_lfs_path), path) + git_cmd = Cmd(git_cmd; windows_verbatim=true) git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep)) end return git_cmd From 895f9aa6ec7fd8a9465695f9ebd11404f2876a2f Mon Sep 17 00:00:00 2001 From: cgarling Date: Sat, 19 Jul 2025 09:04:44 -0400 Subject: [PATCH 09/11] try removing single quotes from cmd.exec --- src/git_function.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/git_function.jl b/src/git_function.jl index e0e325e..f330eec 100644 --- a/src/git_function.jl +++ b/src/git_function.jl @@ -79,12 +79,18 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) git_cmd.env[idx][6:end] end path = vcat(dirname(Git_LFS_jll.git_lfs_path), path) - git_cmd = Cmd(git_cmd; windows_verbatim=true) + git_cmd = clean_cmd(git_cmd) git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep)) end return git_cmd end +function clean_cmd(cmd::Cmd) + # Remove single quotes + exec = map(arg -> replace(arg, "'" => ""), cmd.exec) + return Cmd(Cmd(exec); env=cmd.env, dir=cmd.dir, ignorestatus=cmd.ignorestatus) +end + function git(args::AbstractVector{<:AbstractString}; kwargs...) cmd = git(; kwargs...) append!(cmd.exec, args) From f1d3d3afb84b4e131e79cfd4c8d1f6440f82d5a1 Mon Sep 17 00:00:00 2001 From: cgarling Date: Sat, 19 Jul 2025 09:41:38 -0400 Subject: [PATCH 10/11] revert quote filtering scheme --- src/git_function.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/git_function.jl b/src/git_function.jl index f330eec..b4fd1e6 100644 --- a/src/git_function.jl +++ b/src/git_function.jl @@ -79,18 +79,11 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) git_cmd.env[idx][6:end] end path = vcat(dirname(Git_LFS_jll.git_lfs_path), path) - git_cmd = clean_cmd(git_cmd) - git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep)) + git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep))::Cmd end return git_cmd end -function clean_cmd(cmd::Cmd) - # Remove single quotes - exec = map(arg -> replace(arg, "'" => ""), cmd.exec) - return Cmd(Cmd(exec); env=cmd.env, dir=cmd.dir, ignorestatus=cmd.ignorestatus) -end - function git(args::AbstractVector{<:AbstractString}; kwargs...) cmd = git(; kwargs...) append!(cmd.exec, args) From 981401a8f9086546fd52b1aea7ca33131bec5931 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 24 Jul 2025 14:09:34 +0300 Subject: [PATCH 11/11] refactor --- src/git_function.jl | 50 ++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/git_function.jl b/src/git_function.jl index b4fd1e6..95e48b6 100644 --- a/src/git_function.jl +++ b/src/git_function.jl @@ -22,9 +22,12 @@ julia> run(git(["clone", "https://github.com/JuliaRegistries/General"])) to bypass the parsing of the command string. """ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - git_cmd = @static if Sys.iswindows() - Git_jll.git(; adjust_PATH, adjust_LIBPATH)::Cmd - else + git_cmd = Git_jll.git(; adjust_PATH, adjust_LIBPATH)::Cmd + env_mapping = Dict{String,String}( + "PATH" => _get_cmd_env(git_cmd, "PATH"), + LIBPATH_env => _get_cmd_env(git_cmd, LIBPATH_env), + ) + @static if !Sys.iswindows() root = Git_jll.artifact_dir libexec = joinpath(root, "libexec") @@ -36,7 +39,6 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) ssl_cert = joinpath(dirname(Sys.BINDIR), "share", "julia", "cert.pem") - env_mapping = Dict{String,String}() env_mapping["GIT_EXEC_PATH"] = libexec_git_core env_mapping["GIT_SSL_CAINFO"] = ssl_cert env_mapping["GIT_TEMPLATE_DIR"] = share_git_core_templates @@ -47,15 +49,12 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # more details. env_mapping["JLL_DYLD_FALLBACK_LIBRARY_PATH"] = Git_jll.LIBPATH[] end - - original_cmd = Git_jll.git(; adjust_PATH, adjust_LIBPATH)::Cmd - addenv(original_cmd, env_mapping...)::Cmd end # Use OpenSSH from the JLL: . if !Sys.iswindows() && OpenSSH_jll.is_available() - path = split(get(ENV, "PATH", ""), pathsep) - libpath = split(get(ENV, LIBPATH_env, ""), pathsep) + path = split(get(env_mapping, "PATH", ""), pathsep) + libpath = split(get(env_mapping, LIBPATH_env, ""), pathsep) path = vcat(dirname(OpenSSH_jll.ssh_path), path) libpath = vcat(OpenSSH_jll.LIBPATH_list, libpath) @@ -65,23 +64,20 @@ function git(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) unique!(filter!(!isempty, path)) unique!(filter!(!isempty, libpath)) - git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep), LIBPATH_env => join(libpath, pathsep)) + env_mapping["PATH"] = join(path, pathsep) + env_mapping[LIBPATH_env] = join(libpath, pathsep) end # Add git-lfs if Git_LFS_jll.is_available() - # Read path from git_cmd.env as it can be modified above - idx = findfirst(startswith("PATH="), git_cmd.env) - path = if isnothing(idx) - "" - else - # dropping the `PATH=` part - git_cmd.env[idx][6:end] - end - path = vcat(dirname(Git_LFS_jll.git_lfs_path), path) - git_cmd = addenv(git_cmd, "PATH" => join(path, pathsep))::Cmd + env_mapping["PATH"] = string( + dirname(Git_LFS_jll.git_lfs_path), + pathsep, + get(env_mapping, "PATH", "") + ) end - return git_cmd + + return addenv(git_cmd, env_mapping...)::Cmd end function git(args::AbstractVector{<:AbstractString}; kwargs...) @@ -89,3 +85,15 @@ function git(args::AbstractVector{<:AbstractString}; kwargs...) append!(cmd.exec, args) return cmd end + +# The .env field of a Cmd object is an array of strings in the format +# `$(key)=$(value)` for each environment variable. +function _get_cmd_env(cmd::Cmd, key::AbstractString) + idx = findfirst(startswith("$(key)="), cmd.env) + if isnothing(idx) + return "" + else + # dropping the `$(key)=` part + return cmd.env[idx][(ncodeunits(key)+2):end] + end +end