Skip to content

Commit 3deba3a

Browse files
IanButterworthIanButterworth
committed
make registry download and updates use pidfile locks
1 parent 0bdbb85 commit 3deba3a

File tree

1 file changed

+130
-79
lines changed

1 file changed

+130
-79
lines changed

‎src/Registry/Registry.jl‎

Lines changed: 130 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ using ..Pkg: depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_serv
55
GitTools, get_bool_env
66
using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z
77
using UUIDs, LibGit2, TOML
8+
import ..Pkg.Pidfile
89

910
include("registry_instance.jl")
1011

@@ -163,89 +164,134 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d
163164
populate_known_registries_with_urls!(regs)
164165
regdir = joinpath(depot, "registries")
165166
isdir(regdir) || mkpath(regdir)
166-
registry_urls = pkg_server_registry_urls()
167-
for reg in regs
168-
if reg.path !== nothing && reg.url !== nothing
169-
Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.")
170-
end
171-
url = get(registry_urls, reg.uuid, nothing)
172-
if url !== nothing && registry_read_from_tarball()
173-
tmp = tempname()
174-
try
175-
download_verify(url, nothing, tmp)
176-
catch err
177-
Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))")
178-
end
179-
if reg.name === nothing
180-
# Need to look up the registry name here
181-
reg_unc = uncompress_registry(tmp)
182-
reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String
183-
end
184-
mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true)
185-
_hash = pkg_server_url_hash(url)
186-
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz")
187-
open(joinpath(regdir, reg.name * ".toml"), "w") do io
188-
TOML.print(io, reg_info)
167+
Pidfile.mkpidlock(joinpath(regdir, ".pidlock")) do # only allow one julia process to download registries at a time
168+
registry_urls = pkg_server_registry_urls()
169+
for reg in regs
170+
if reg.path !== nothing && reg.url !== nothing
171+
Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.")
189172
end
190-
else
191-
mktempdir() do tmp
192-
if reg.path !== nothing && reg.linked == true # symlink to local source
193-
registry = Registry.RegistryInstance(reg.path)
194-
regpath = joinpath(regdir, registry.name)
195-
printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
196-
isdir(dirname(regpath)) || mkpath(dirname(regpath))
197-
symlink(reg.path, regpath)
198-
isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
199-
registry = Registry.RegistryInstance(regpath)
200-
printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
201-
return
202-
elseif url !== nothing && registry_use_pkg_server()
203-
# download from Pkg server
204-
try
205-
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
206-
catch err
207-
Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))")
208-
end
209-
tree_info_file = joinpath(tmp, ".tree_info.toml")
210-
hash = pkg_server_url_hash(url)
211-
write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
212-
elseif reg.path !== nothing # copy from local source
213-
printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
214-
isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
215-
registry = Registry.RegistryInstance(reg.path)
216-
regpath = joinpath(regdir, registry.name)
217-
cp(reg.path, regpath; force=true) # has to be cp given we're copying
218-
printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
219-
return
220-
elseif reg.url !== nothing # clone from url
221-
repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
222-
LibGit2.close(repo)
223-
else
224-
Pkg.Types.pkgerror("no path or url specified for registry")
173+
url = get(registry_urls, reg.uuid, nothing)
174+
if url !== nothing && registry_read_from_tarball()
175+
tmp = tempname()
176+
try
177+
download_verify(url, nothing, tmp)
178+
catch err
179+
Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))")
180+
end
181+
if reg.name === nothing
182+
# Need to look up the registry name here
183+
reg_unc = uncompress_registry(tmp)
184+
reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String
225185
end
226-
# verify that the clone looks like a registry
227-
if !isfile(joinpath(tmp, "Registry.toml"))
228-
Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
186+
mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true)
187+
_hash = pkg_server_url_hash(url)
188+
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(_hash), "path" => reg.name * ".tar.gz")
189+
open(joinpath(regdir, reg.name * ".toml"), "w") do io
190+
TOML.print(io, reg_info)
229191
end
230-
registry = Registry.RegistryInstance(tmp)
231-
regpath = joinpath(regdir, registry.name)
232-
# copy to `depot`
233-
ispath(dirname(regpath)) || mkpath(dirname(regpath))
234-
if isfile(joinpath(regpath, "Registry.toml"))
235-
existing_registry = Registry.RegistryInstance(regpath)
236-
if registry.uuid == existing_registry.uuid
237-
println(io,
238-
"Registry `$(registry.name)` already exists in `$(Base.contractuser(regpath))`.")
192+
else
193+
mktempdir() do tmp
194+
if reg.path !== nothing && reg.linked == true # symlink to local source
195+
registry = Registry.RegistryInstance(reg.path)
196+
regpath = joinpath(regdir, registry.name)
197+
printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
198+
isdir(dirname(regpath)) || mkpath(dirname(regpath))
199+
symlink(reg.path, regpath)
200+
isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
201+
registry = Registry.RegistryInstance(regpath)
202+
printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
203+
return
204+
elseif url !== nothing && registry_use_pkg_server()
205+
# download from Pkg server
206+
try
207+
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
208+
catch err
209+
Pkg.Types.pkgerror("could not download $url \nException: $(sprint(showerror, err))")
210+
end
211+
tree_info_file = joinpath(tmp, ".tree_info.toml")
212+
hash = pkg_server_url_hash(url)
213+
write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
214+
elseif reg.path !== nothing # copy from local source
215+
printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
216+
isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
217+
registry = Registry.RegistryInstance(reg.path)
218+
regpath = joinpath(regdir, registry.name)
219+
cp(reg.path, regpath; force=true) # has to be cp given we're copying
220+
printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
221+
return
222+
elseif reg.url !== nothing # clone from url
223+
repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
224+
LibGit2.close(repo)
239225
else
240-
throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
241-
"existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
242-
"To install it you can clone it manually into e.g. " *
243-
"`$(Base.contractuser(joinpath(regdir, registry.name*"-2")))`."))
226+
Pkg.Types.pkgerror("no path or url specified for registry")
227+
end
228+
mv(tmp, joinpath(regdir, reg.name * ".tar.gz"); force=true)
229+
hash = pkg_server_url_hash(url)
230+
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "path" => reg.name * ".tar.gz")
231+
open(joinpath(regdir, reg.name * ".toml"), "w") do io
232+
TOML.print(io, reg_info)
233+
end
234+
else
235+
mktempdir() do tmp
236+
if reg.path !== nothing && reg.linked == true # symlink to local source
237+
registry = Registry.RegistryInstance(reg.path)
238+
regpath = joinpath(regdir, registry.name)
239+
printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
240+
isdir(dirname(regpath)) || mkpath(dirname(regpath))
241+
symlink(reg.path, regpath)
242+
isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
243+
registry = Registry.RegistryInstance(regpath)
244+
printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
245+
return
246+
elseif url !== nothing && registry_use_pkg_server()
247+
# download from Pkg server
248+
try
249+
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
250+
catch err
251+
Pkg.Types.pkgerror("could not download $url")
252+
end
253+
tree_info_file = joinpath(tmp, ".tree_info.toml")
254+
hash = pkg_server_url_hash(url)
255+
write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
256+
elseif reg.path !== nothing # copy from local source
257+
printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
258+
isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
259+
registry = Registry.RegistryInstance(reg.path)
260+
regpath = joinpath(regdir, registry.name)
261+
cp(reg.path, regpath; force=true) # has to be cp given we're copying
262+
printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
263+
return
264+
elseif reg.url !== nothing # clone from url
265+
repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
266+
LibGit2.close(repo)
267+
else
268+
Pkg.Types.pkgerror("no path or url specified for registry")
269+
end
270+
# verify that the clone looks like a registry
271+
if !isfile(joinpath(tmp, "Registry.toml"))
272+
Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
273+
end
274+
registry = Registry.RegistryInstance(tmp)
275+
regpath = joinpath(regdir, registry.name)
276+
# copy to `depot`
277+
ispath(dirname(regpath)) || mkpath(dirname(regpath))
278+
if isfile(joinpath(regpath, "Registry.toml"))
279+
existing_registry = Registry.RegistryInstance(regpath)
280+
if registry.uuid == existing_registry.uuid
281+
println(io,
282+
"Registry `$(registry.name)` already exists in `$(Base.contractuser(regpath))`.")
283+
else
284+
throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
285+
"existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
286+
"To install it you can clone it manually into e.g. " *
287+
"`$(Base.contractuser(joinpath(regdir, registry.name*"-2")))`."))
288+
end
289+
elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true
290+
# if the dir doesn't exist, or exists but doesn't contain a Registry.toml
291+
mv(tmp, regpath, force=true)
292+
printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
293+
end
244294
end
245-
elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true
246-
# if the dir doesn't exist, or exists but doesn't contain a Registry.toml
247-
mv(tmp, regpath, force=true)
248-
printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
249295
end
250296
end
251297
end
@@ -348,7 +394,8 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(),
348394
for reg in unique(r -> r.uuid, find_installed_registries(io, regs); seen=Set{UUID}())
349395
let reg=reg, errors=errors
350396
regpath = pathrepr(reg.path)
351-
let regpath=regpath
397+
Pidfile.mkpidlock(joinpath(dirname(reg.path), ".pidlock")) do # only allow one julia process to update a given registry at a time
398+
let regpath=regpath
352399
if reg.tree_info !== nothing
353400
printpkgstyle(io, :Updating, "registry at " * regpath)
354401
old_hash = reg.tree_info
@@ -372,6 +419,10 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(),
372419
if isdir(reg.path)
373420
Base.rm(reg.path; recursive=true, force=true)
374421
end
422+
# If we have an uncompressed Pkg server registry, remove it and get the compressed version
423+
if isdir(reg.path)
424+
Base.rm(reg.path; recursive=true, force=true)
425+
end
375426
registry_path = dirname(reg.path)
376427
mv(tmp, joinpath(registry_path, reg.name * ".tar.gz"); force=true)
377428
hash = pkg_server_url_hash(url)

0 commit comments

Comments
 (0)