diff --git a/CHANGELOG.md b/CHANGELOG.md index 26d4d4f7a6..b4dbdb9b7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ Pkg v1.9 Release Notes to the directory of the package being tested. Previously the `--code-coverage=user` option was used, which tracked files in all code outside of Core & Base, i.e. all stdlibs and all user packages, which often made running tests with code coverage a lot slower ([#3021]). +- Writes to `manifest_usage.toml` and Registry downloads/updates are now protected from process concurrency clashes via + a pidfile lock ([#2793]). Pkg v1.8 Release Notes ====================== @@ -59,6 +61,7 @@ Pkg v1.7 Release Notes [#2580]: https://github.com/JuliaLang/Pkg.jl/issues/2580 [#2689]: https://github.com/JuliaLang/Pkg.jl/issues/2689 [#2702]: https://github.com/JuliaLang/Pkg.jl/issues/2702 +[#2793]: https://github.com/JuliaLang/Pkg.jl/issues/2793 [#2815]: https://github.com/JuliaLang/Pkg.jl/issues/2815 [#2906]: https://github.com/JuliaLang/Pkg.jl/issues/2906 [#2933]: https://github.com/JuliaLang/Pkg.jl/issues/2933 diff --git a/Project.toml b/Project.toml index a414d40d24..995e96b0a5 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ version = "1.8.0" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" diff --git a/ext/HistoricaStdlibGenerator/Manifest.toml b/ext/HistoricaStdlibGenerator/Manifest.toml index aed2488356..b0676907bf 100644 --- a/ext/HistoricaStdlibGenerator/Manifest.toml +++ b/ext/HistoricaStdlibGenerator/Manifest.toml @@ -1,140 +1,160 @@ # This file is machine-generated - editing it directly is not advised -[[ArgTools]] +julia_version = "1.9.0-DEV" +manifest_format = "2.0" +project_hash = "136cd253a89b81157598e253c00b82898978de50" + +[[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" -[[Artifacts]] +[[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -[[Base64]] +[[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -[[Dates]] +[[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -[[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] -git-tree-sha1 = "5a19556930ddb432b1094e0796911c16282c517f" +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -[[InteractiveUtils]] +[[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -[[JSON3]] +[[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "StructTypes", "UUIDs"] -git-tree-sha1 = "961ef1c3e5c8a595d5bec270a9007429ef12ed10" +git-tree-sha1 = "7d58534ffb62cd947950b3aa9b993e63307a6125" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" -version = "1.5.1" +version = "1.9.2" -[[LibCURL]] +[[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" -[[LibCURL_jll]] +[[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.81.0+0" -[[LibGit2]] +[[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -[[LibSSH2_jll]] +[[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" -[[Libdl]] +[[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -[[Logging]] +[[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -[[Markdown]] +[[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -[[MbedTLS_jll]] +[[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.0+0" -[[Mmap]] +[[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -[[MozillaCACerts_jll]] +[[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.2.1" -[[NetworkOptions]] +[[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" -[[Parsers]] +[[deps.Parsers]] deps = ["Dates"] -git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714" +git-tree-sha1 = "13468f237353112a01b2d6b32f3d0f80219944aa" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "1.0.15" +version = "2.2.2" -[[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] path = "../.." uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78e" -version = "1.5.0" +version = "1.8.0" -[[Printf]] +[[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -[[REPL]] +[[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -[[Random]] -deps = ["Serialization"] +[[deps.Random]] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -[[SHA]] +[[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" -[[Scratch]] +[[deps.Scratch]] deps = ["Dates"] -git-tree-sha1 = "ad4b278adb62d185bbcb6864dc24959ab0627bf6" +git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.0.3" +version = "1.1.0" -[[Serialization]] +[[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -[[Sockets]] +[[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -[[StructTypes]] +[[deps.StructTypes]] deps = ["Dates", "UUIDs"] -git-tree-sha1 = "d94235fcdc4a09649f263365c5f7e4ed4ba6ed34" +git-tree-sha1 = "d24a825a95a6d98c385001212dc9020d609f2d4f" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" -version = "1.2.1" +version = "1.8.1" -[[TOML]] +[[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" -[[Tar]] +[[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" -[[UUIDs]] +[[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -[[Unicode]] +[[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -[[Zlib_jll]] +[[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.12+1" -[[nghttp2_jll]] +[[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.41.0+1" -[[p7zip_jll]] +[[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "16.2.1+1" diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index c401fd94ba..0be2229e96 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -5,6 +5,7 @@ using ..Pkg: depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_serv GitTools, get_bool_env using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z using UUIDs, LibGit2, TOML +import FileWatching include("registry_instance.jl") @@ -163,6 +164,8 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d populate_known_registries_with_urls!(regs) regdir = joinpath(depot, "registries") isdir(regdir) || mkpath(regdir) + # only allow one julia process to download and install registries at a time + FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do registry_urls = pkg_server_registry_urls() for reg in regs if reg.path !== nothing && reg.url !== nothing @@ -250,6 +253,7 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d end end end + end # mkpidlock return nothing end @@ -342,7 +346,12 @@ Pkg.Registry.update(RegistrySpec(uuid = "23338594-aafe-5451-b93e-139f81909106")) update(reg::Union{String,RegistrySpec}; kwargs...) = update([reg]; kwargs...) update(regs::Vector{String}; kwargs...) = update([RegistrySpec(name = name) for name in regs]; kwargs...) function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), force::Bool=true) - isempty(regs) && (regs = reachable_registries(; depots=depots1())) + depot = depots1() + isempty(regs) && (regs = reachable_registries(; depots=depot)) + regdir = joinpath(depot, "registries") + isdir(regdir) || mkpath(regdir) + # only allow one julia process to update registries at a time + FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do errors = Tuple{String, String}[] registry_urls = pkg_server_registry_urls() for reg in unique(r -> r.uuid, find_installed_registries(io, regs); seen=Set{UUID}()) @@ -448,6 +457,7 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), end @error warn_str end + end # mkpidlock return end diff --git a/src/Types.jl b/src/Types.jl index e53612cecc..0c49c92727 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -15,6 +15,7 @@ import ..Pkg, ..Registry import ..Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server, stdlib_dir, stdlib_path, isurl, stderr_f, RESPECT_SYSIMAGE_VERSIONS import Base.BinaryPlatforms: Platform using ..Pkg.Versions +import FileWatching import Base: SHA1 using SHA @@ -484,16 +485,31 @@ function write_env_usage(source_file::AbstractString, usage_filepath::AbstractSt # Ensure that log dir exists !ispath(logdir()) && mkpath(logdir()) - # Generate entire entry as a string first - entry = sprint() do io - TOML.print(io, Dict(source_file => [Dict("time" => now())])) - end - - # Append entry to log file in one chunk usage_file = joinpath(logdir(), usage_filepath) - open(usage_file, append=true) do io - write(io, entry) + timestamp = now() + + ## Atomically write usage file using process id locking + FileWatching.mkpidlock(usage_file * ".pid", stale_age = 3) do + usage = if isfile(usage_file) + TOML.parsefile(usage_file) + else + Dict{String, Any}() + end + + # record new usage + usage[source_file] = [Dict("time" => timestamp)] + + # keep only latest usage info + for k in keys(usage) + times = map(d -> Dates.DateTime(d["time"]), usage[k]) + usage[k] = [Dict("time" => maximum(times))] + end + + open(usage_file, "w") do io + TOML.print(io, usage, sorted=true) + end end + return end function read_package(path::String) diff --git a/test/pkg.jl b/test/pkg.jl index a5ab027eca..6f0a383ed5 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -363,6 +363,56 @@ temp_pkg_dir() do project_path @test any(x -> startswith(x, manifest), keys(usage)) end + @testset "test atomicity of write_env_usage with $(Sys.CPU_THREADS) parallel processes" begin + tasks = Task[] + iobs = IOBuffer[] + Sys.CPU_THREADS == 1 && error("Cannot test for atomic usage log file interaction effectively with only Sys.CPU_THREADS=1") + run(`$(Base.julia_cmd()) --project="$(pkgdir(Pkg))" -e "import Pkg"`) # to precompile Pkg given we're in a different depot + flag_start_dir = tempdir() # once n=Sys.CPU_THREADS files are in here, the processes can proceed to the concurrent test + flag_end_file = tempname() # use creating this file as a way to stop the processes early if an error happens + for i in 1:Sys.CPU_THREADS + iob = IOBuffer() + t = @async run(pipeline(`$(Base.julia_cmd()[1]) --project="$(pkgdir(Pkg))" + -e "import Pkg; + Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true; + Pkg.activate(temp = true); + Pkg.add(\"Random\", io = devnull); + touch(tempname(raw\"$flag_start_dir\")) # file marker that first part has finished + while length(readdir(raw\"$flag_start_dir\")) < $(Sys.CPU_THREADS) + # sync all processes to start at the same time + sleep(0.1) + end + @async begin + sleep(15) + touch(raw\"$flag_end_file\") + end + i = 0 + while !isfile(raw\"$flag_end_file\") + global i += 1 + try + Pkg.Types.EnvCache() + catch + touch(raw\"$flag_end_file\") + println(stderr, \"Errored after $i iterations\") + rethrow() + end + yield() + end"`, + stderr = iob, stdout = devnull)) + push!(tasks, t) + push!(iobs, iob) + end + for i in eachindex(tasks) + try + fetch(tasks[i]) # If any of these failed it will throw when fetched + catch + print(String(take!(iobs[i]))) + break + end + end + @test any(istaskfailed, tasks) == false + end + @testset "adding nonexisting packages" begin nonexisting_pkg = randstring(14) @test_throws PkgError Pkg.add(nonexisting_pkg)