Disallow precompiling Pkg in Pkg tests (#4176)

Author: IanButterworth

ext/REPLExt/REPLExt.jl

@@ -1,5 +1,9 @@
 module REPLExt
 
+if Base.get_bool_env("JULIA_PKG_DISALLOW_PKG_PRECOMPILATION", false) == true
+    error("Precompililing Pkg extension REPLExt is disallowed. JULIA_PKG_DISALLOW_PKG_PRECOMPILATION=$(ENV["JULIA_PKG_DISALLOW_PKG_PRECOMPILATION"])")
+end
+
 using Markdown, UUIDs, Dates
 
 import REPL

src/Pkg.jl

@@ -2,6 +2,11 @@
 
 module Pkg
 
+# In Pkg tests we want to avoid Pkg being re-precompiled by subprocesses, so this is enabled in the test suite
+if Base.get_bool_env("JULIA_PKG_DISALLOW_PKG_PRECOMPILATION", false) == true
+    error("Precompililing Pkg is disallowed. JULIA_PKG_DISALLOW_PKG_PRECOMPILATION=$(ENV["JULIA_PKG_DISALLOW_PKG_PRECOMPILATION"])")
+end
+
 if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@max_methods"))
     @eval Base.Experimental.@max_methods 1
 end

test/new.jl

@@ -141,14 +141,6 @@ Pkg._auto_gc_enabled[] = false
     end
 end
 
-function copy_this_pkg_cache(new_depot)
-    source = joinpath(Base.DEPOT_PATH[1], "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg")
-    isdir(source) || return # doesn't exist if using shipped Pkg (e.g. Julia CI)
-    dest = joinpath(new_depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "Pkg")
-    mkpath(dirname(dest))
-    cp(source, dest)
-end
-
 function kill_with_info(p)
     if Sys.islinux()
         SIGINFO = 10

test/pkg.jl

@@ -401,12 +401,12 @@ temp_pkg_dir() do project_path
         Sys.CPU_THREADS == 1 && error("Cannot test for atomic usage log file interaction effectively with only Sys.CPU_THREADS=1")
         # Precompile Pkg given we're in a different depot
         # and make sure the General registry is installed
-        Utils.show_output_if_command_errors(`$(Base.julia_cmd()[1]) --project="$(pkgdir(Pkg))" -e "import Pkg; isempty(Pkg.Registry.reachable_registries()) && Pkg.Registry.add()"`)
+        Utils.show_output_if_command_errors(`$(Base.julia_cmd()) --project="$(pkgdir(Pkg))" -e "import Pkg; isempty(Pkg.Registry.reachable_registries()) && Pkg.Registry.add()"`)
         flag_start_dir = tempdir() # once n=Sys.CPU_THREADS files are in here, the processes can proceed to the concurrent test
         flag_end_file = tempname() # use creating this file as a way to stop the processes early if an error happens
         for i in 1:Sys.CPU_THREADS
             iob = IOBuffer()
-            t = @async run(pipeline(`$(Base.julia_cmd()[1]) --project="$(pkgdir(Pkg))"
+            t = @async run(pipeline(`$(Base.julia_cmd()) --project="$(pkgdir(Pkg))"
                 -e "import Pkg;
                 Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true;
                 Pkg.activate(temp = true);

test/repl.jl

@@ -735,14 +735,16 @@ end
 end
 
 @testset "JuliaLang/julia #55850" begin
-    tmp_55850 = mktempdir()
-    tmp_sym_link = joinpath(tmp_55850, "sym")
-    symlink(tmp_55850, tmp_sym_link; dir_target=true)
-    depot_path = join([tmp_sym_link, Base.DEPOT_PATH...], Sys.iswindows() ? ";" : ":")
-    # include the symlink in the depot path and include the regular default depot so we don't precompile this Pkg again
-    withenv("JULIA_DEPOT_PATH" => depot_path, "JULIA_LOAD_PATH" => nothing) do
-        prompt = readchomp(`$(Base.julia_cmd()[1]) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`)
-        @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> "
+    mktempdir() do tmp
+        copy_this_pkg_cache(tmp)
+        tmp_sym_link = joinpath(tmp, "sym")
+        symlink(tmp, tmp_sym_link; dir_target=true)
+        depot_path = tmp_sym_link * (Sys.iswindows() ? ";" : ":")
+        # include the symlink in the depot path and include the regular default depot so we don't precompile this Pkg again
+        withenv("JULIA_DEPOT_PATH" => depot_path, "JULIA_LOAD_PATH" => nothing) do
+            prompt = readchomp(`$(Base.julia_cmd()) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`)
+            @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> "
+        end
     end
 end
 

test/runtests.jl

@@ -20,6 +20,7 @@ if realpath(dirname(dirname(Base.pathof(Pkg)))) != realpath(dirname(@__DIR__))
 end
 
 ENV["JULIA_PKG_PRECOMPILE_AUTO"]=0
+ENV["JULIA_PKG_DISALLOW_PKG_PRECOMPILATION"]=1
 
 logdir = get(ENV, "JULIA_TEST_VERBOSE_LOGS_DIR", nothing)
 ### Send all Pkg output to a file called Pkg.log

test/utils.jl

@@ -11,7 +11,7 @@ using UUIDs
 export temp_pkg_dir, cd_tempdir, isinstalled, write_build, with_current_env,
        with_temp_env, with_pkg_env, git_init_and_commit, copy_test_package,
        git_init_package, add_this_pkg, TEST_SIG, TEST_PKG, isolate, LOADED_DEPOT,
-       list_tarball_files, recursive_rm_cov_files
+       list_tarball_files, recursive_rm_cov_files, copy_this_pkg_cache
 
 const CACHE_DIRECTORY = realpath(mktempdir(; cleanup = true))
 
@@ -22,6 +22,17 @@ const REGISTRY_DIR = joinpath(REGISTRY_DEPOT, "registries", "General")
 
 const GENERAL_UUID = UUID("23338594-aafe-5451-b93e-139f81909106")
 
+function copy_this_pkg_cache(new_depot)
+    for p in ("Pkg", "REPLExt")
+        subdir = joinpath("compiled", "v$(VERSION.major).$(VERSION.minor)")
+        source = joinpath(Base.DEPOT_PATH[1], subdir, p)
+        isdir(source) || continue # doesn't exist if using shipped Pkg (e.g. Julia CI)
+        dest = joinpath(new_depot, subdir, p)
+        mkpath(dirname(dest))
+        cp(source, dest)
+    end
+end
+
 function check_init_reg()
     isfile(joinpath(REGISTRY_DIR, "Registry.toml")) && return
     mkpath(REGISTRY_DIR)