Fix out-of-tree docs build (#58529)

The `doc` build had only limited out-of-tree support. In particular, it
was relying on creating in-tree `stdlib` symbolic links (due to a
missing feature in Documenter), as well as pointing external stdlibs to
the in-tree references rather than the out-of-tree ones. Fix this by
building up the reading of the buildroot and splitting `@__DIR__`
references appropriately, depending on whether they refer to the source
or the output. Fixes #53039.

Author: Keno

doc/Makefile

@@ -30,6 +30,13 @@ $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt:
 	@mkdir -p "$(SRCCACHE)"
 	$(JLDOWNLOAD) "$@" https://www.unicode.org/Public/$(UNICODE_DATA_VERSION)/ucd/UnicodeData.txt
 
+# NEWS.md and stdlib are in-tree build artifacts - don't link them for oot builds.
+DOC_FILES=$(filter-out NEWS.md stdlib,$(notdir $(wildcard $(SRCDIR)/src/*)))
+src/%:
+	@mkdir -p src
+	ln -s $(SRCDIR)/src/$* $@
+src: $(addprefix src/,$(DOC_FILES))
+
 deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt
 	$(JLCHECKSUM) "$<"
 	cp "$<" UnicodeData.txt

doc/make.jl

@@ -1,9 +1,13 @@
+let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS)
+    global const buildrootdoc = i === nothing ? (@__DIR__) : joinpath(first(match(r, ARGS[i]).captures), "doc")
+end
+
 # Install dependencies needed to build the documentation.
 Base.ACTIVE_PROJECT[] = nothing
 empty!(LOAD_PATH)
 push!(LOAD_PATH, @__DIR__, "@stdlib")
 empty!(DEPOT_PATH)
-push!(DEPOT_PATH, joinpath(@__DIR__, "deps"))
+push!(DEPOT_PATH, joinpath(buildrootdoc, "deps"))
 push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia"))
 using Pkg
 Pkg.instantiate()
@@ -26,7 +30,7 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest)
 const STDLIB_DOCS = []
 const STDLIB_DIR = Sys.STDLIB
 const EXT_STDLIB_DOCS = ["Pkg"]
-cd(joinpath(@__DIR__, "src")) do
+cd(joinpath(buildrootdoc, "src")) do
     Base.rm("stdlib"; recursive=true, force=true)
     mkdir("stdlib")
     for dir in readdir(STDLIB_DIR)
@@ -70,7 +74,8 @@ function parse_stdlib_version_file(path)
 end
 # This generates the value that will be passed to the `remotes` argument of makedocs(),
 # by looking through all *.version files in stdlib/.
-documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "stdlib"))
+documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "stdlib")),
+                                stdlib_build_dir = joinpath(buildrootdoc, "..", "stdlib")
     # Get a list of all *.version files in stdlib/..
     version_files = filter(readdir(stdlib_dir)) do fname
         isfile(joinpath(stdlib_dir, fname)) && endswith(fname, ".version")
@@ -97,7 +102,7 @@ documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "
             versionfile[sha_key]
         end
         # Construct the absolute (local) path to the stdlib package's root directory
-        package_root_dir = joinpath(stdlib_dir, "$(package)-$(package_sha)")
+        package_root_dir = joinpath(stdlib_build_dir, "$(package)-$(package_sha)")
         # Documenter needs package_root_dir to exist --- it's just a sanity check it does on the remotes= keyword.
         # In normal (local) builds, this will be the case, since the Makefiles will have unpacked the standard
         # libraries. However, on CI we do this thing where we actually build docs in a clean worktree, just
@@ -127,7 +132,7 @@ function generate_markdown(basename)
     @assert length(splitted) == 2
     replaced_links = replace(splitted[1], r"\[\#([0-9]*?)\]" => s"[#\g<1>](https://github.com/JuliaLang/julia/issues/\g<1>)")
     write(
-        joinpath(@__DIR__, "src", "$basename.md"),
+        joinpath(buildrootdoc, "src", "$basename.md"),
         """
         ```@meta
         EditURL = "https://github.com/JuliaLang/julia/blob/master/$basename.md"
@@ -281,7 +286,7 @@ end
 
 const use_revise = "revise=true" in ARGS
 if use_revise
-    let revise_env = joinpath(@__DIR__, "deps", "revise")
+    let revise_env = joinpath(buildrootdoc, "deps", "revise")
         Pkg.activate(revise_env)
         Pkg.add("Revise"; preserve=Pkg.PRESERVE_NONE)
         Base.ACTIVE_PROJECT[] = nothing
@@ -351,10 +356,6 @@ DocMeta.setdocmeta!(
     recursive=true, warn=false,
 )
 
-let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS)
-    global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures)
-end
-
 const format = if render_pdf
     Documenter.LaTeX(
         platform = "texplatform=docker" in ARGS ? "docker" : "native"
@@ -377,8 +378,9 @@ else
     )
 end
 
-const output_path = joinpath(buildroot, "doc", "_build", (render_pdf ? "pdf" : "html"), "en")
+const output_path = joinpath(buildrootdoc, "_build", (render_pdf ? "pdf" : "html"), "en")
 makedocs(
+    source    = joinpath(buildrootdoc, "src"),
     build     = output_path,
     modules   = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...],
     clean     = true,
@@ -479,7 +481,7 @@ if "deploy" in ARGS
     deploydocs(
         repo = "github.com/JuliaLang/docs.julialang.org.git",
         deploy_config = BuildBotConfig(),
-        target = joinpath(buildroot, "doc", "_build", "html", "en"),
+        target = joinpath(buildrootdoc, "_build", "html", "en"),
         dirname = "en",
         devurl = devurl,
         versions = Versions(["v#.#", devurl => devurl]),