Make Compiler an independent package (that lives in the julia repo) (JuliaLang#56409)

This is a further extension to JuliaLang#56128 to make the compiler into a proper
stdlib, useable outside of `Base` as `using Compiler` in the same way
that `JuliaSyntax` works already. There's a few remaining questions
around how loading works, but mechanically, this PR is complete. For
those remaining questions, I'll probably put up a separate PR that would
migrate JuliaSyntax to it and then simply adopt that here once we've
figured out the correct semantics.

Author: Keno

Compiler/Project.toml

@@ -0,0 +1,3 @@
+name = "Compiler"
+uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1"
+version = "0.0.1"

Compiler/src/Compiler.jl

@@ -0,0 +1,194 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# When generating an incremental precompile file, we first check whether we
+# already have a copy of this *exact* code in the system image. If so, we
+# simply generates a pkgimage that has the dependency edges we recorded in
+# the system image and simply returns that copy of the compiler. If not,
+# we proceed to load/precompile this as an ordinary package.
+if isdefined(Base, :generating_output) && Base.generating_output(true) &&
+        Base.samefile(Base._compiler_require_dependencies[1][2], @eval @__FILE__) &&
+        !Base.any_includes_stale(
+            map(Base.CacheHeaderIncludes, Base._compiler_require_dependencies),
+            "sysimg", nothing)
+
+    Base.prepare_compiler_stub_image!()
+    append!(Base._require_dependencies, Base._compiler_require_dependencies)
+    # There isn't much point in precompiling native code - downstream users will
+    # specialize their own versions of the compiler code and we don't activate
+    # the compiler by default anyway, so let's save ourselves some disk space.
+    ccall(:jl_suppress_precompile, Cvoid, (Cint,), 1)
+
+else
+
+@eval baremodule Compiler
+
+# Needs to match UUID defined in Project.toml
+ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Compiler,
+    (0x807dbc54_b67e_4c79, 0x8afb_eafe4df6f2e1))
+
+using Core.Intrinsics, Core.IR
+
+import Core: print, println, show, write, unsafe_write,
+             _apply_iterate, svec, apply_type, Builtin, IntrinsicFunction,
+             MethodInstance, CodeInstance, MethodTable, MethodMatch, PartialOpaque,
+             TypeofVararg, Core, SimpleVector, donotdelete, compilerbarrier,
+             memoryref_isassigned, memoryrefnew, memoryrefoffset, memoryrefget,
+             memoryrefset!, typename
+
+using Base
+using Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @_gc_preserve_end, RefValue,
+    @nospecializeinfer, @_foldable_meta, fieldindex, is_function_def, indexed_iterate, isexpr, methods,
+    get_world_counter, JLOptions, _methods_by_ftype, unwrap_unionall, cconvert, unsafe_convert,
+    issingletontype, isType, rewrap_unionall, has_free_typevars, isvarargtype, hasgenerator,
+    IteratorSize, SizeUnknown, _array_for, Bottom, generating_output, diff_names,
+    ismutationfree, NUM_EFFECTS_OVERRIDES, _NAMEDTUPLE_NAME, datatype_fieldtypes,
+    argument_datatype, isfieldatomic, unwrapva, iskindtype, _bits_findnext, copy_exprargs,
+    Generator, Filter, ismutabletypename, isvatuple, datatype_fieldcount,
+    isconcretedispatch, isdispatchelem, datatype_layoutsize,
+    datatype_arrayelem, unionlen, isidentityfree, _uniontypes, uniontypes, OneTo, Callable,
+    DataTypeFieldDesc, datatype_nfields, datatype_pointerfree, midpoint, is_valid_intrinsic_elptr,
+    allocatedinline, isbitsunion, widen_diagonal, unconstrain_vararg_length,
+    rename_unionall, may_invoke_generator, is_meta_expr_head, is_meta_expr, quoted,
+    specialize_method, hasintersect, is_nospecializeinfer, is_nospecialized,
+    get_nospecializeinfer_sig, tls_world_age, uniontype_layout, kwerr,
+    moduleroot, is_file_tracked, decode_effects_override, lookup_binding_partition,
+    is_some_imported, binding_kind, is_some_guard, is_some_const_binding, partition_restriction,
+    BINDING_KIND_GLOBAL, structdiff
+using Base.Order
+import Base: getindex, setindex!, length, iterate, push!, isempty, first, convert, ==,
+    copy, popfirst!, in, haskey, resize!, copy!, append!, last, get!, size,
+    get, iterate, findall, min_world, max_world, _topmod
+
+const getproperty = Core.getfield
+const setproperty! = Core.setfield!
+const swapproperty! = Core.swapfield!
+const modifyproperty! = Core.modifyfield!
+const replaceproperty! = Core.replacefield!
+const _DOCS_ALIASING_WARNING = ""
+
+ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Compiler, false)
+
+eval(x) = Core.eval(Compiler, x)
+eval(m, x) = Core.eval(m, x)
+
+function include(x::String)
+    if !isdefined(Base, :end_base_include)
+        # During bootstrap, all includes are relative to `base/`
+        x = Base.strcat(Base.strcat(Base.BUILDROOT, "../usr/share/julia/Compiler/src/"), x)
+    end
+    Base.include(Compiler, x)
+end
+
+function include(mod::Module, x::String)
+    if !isdefined(Base, :end_base_include)
+        x = Base.strcat(Base.strcat(Base.BUILDROOT, "../usr/share/julia/Compiler/src/"), x)
+    end
+    Base.include(mod, x)
+end
+
+
+macro _boundscheck() Expr(:boundscheck) end
+
+# These types are used by reflection.jl and expr.jl too, so declare them here.
+# Note that `@assume_effects` is available only after loading namedtuple.jl.
+abstract type MethodTableView end
+abstract type AbstractInterpreter end
+
+function return_type end
+function is_return_type(Core.@nospecialize(f))
+    f === return_type && return true
+    if isdefined(Base, :Compiler) && Compiler !== Base.Compiler
+        # Also model the return_type function of the builtin Compiler the same.
+        # This isn't completely sound. We don't actually have any idea what the
+        # base compiler will do at runtime. In the fullness of time, we should
+        # re-work the semantics to make the cache primary and thus avoid having
+        # to reason about what the compiler may do at runtime, but we're not
+        # fully there yet.
+        return f === Base.Compiler.return_type
+    end
+    return false
+end
+
+include("sort.jl")
+
+# We don't include some.jl, but this definition is still useful.
+something(x::Nothing, y...) = something(y...)
+something(x::Any, y...) = x
+
+############
+# compiler #
+############
+
+baremodule BuildSettings
+using Core: ARGS, include
+using ..Compiler: >, getindex, length
+
+global MAX_METHODS::Int = 3
+
+if length(ARGS) > 2 && ARGS[2] === "--buildsettings"
+    include(BuildSettings, ARGS[3])
+end
+end
+
+if false
+    import Base: Base, @show
+else
+    macro show(ex...)
+        blk = Expr(:block)
+        for s in ex
+            push!(blk.args, :(println(stdout, $(QuoteNode(s)), " = ",
+                                              begin local value = $(esc(s)) end)))
+        end
+        isempty(ex) || push!(blk.args, :value)
+        blk
+    end
+end
+
+include("cicache.jl")
+include("methodtable.jl")
+include("effects.jl")
+include("types.jl")
+include("utilities.jl")
+include("validation.jl")
+
+include("ssair/basicblock.jl")
+include("ssair/domtree.jl")
+include("ssair/ir.jl")
+include("ssair/tarjan.jl")
+
+include("abstractlattice.jl")
+include("stmtinfo.jl")
+include("inferenceresult.jl")
+include("inferencestate.jl")
+
+include("typeutils.jl")
+include("typelimits.jl")
+include("typelattice.jl")
+include("tfuncs.jl")
+
+include("abstractinterpretation.jl")
+include("typeinfer.jl")
+include("optimize.jl")
+
+include("bootstrap.jl")
+include("reflection_interface.jl")
+
+if isdefined(Base, :IRShow)
+    @eval module IRShow
+        import ..Compiler
+        using Core.IR
+        using ..Base
+        import .Compiler: IRCode, CFG, scan_ssa_use!,
+            isexpr, compute_basic_blocks, block_for_inst, IncrementalCompact,
+            Effects, ALWAYS_TRUE, ALWAYS_FALSE, DebugInfoStream, getdebugidx,
+            VarState, InvalidIRError, argextype, widenconst, singleton_type,
+            sptypes_from_meth_instance, EMPTY_SPTYPES, InferenceState,
+            NativeInterpreter, CachedMethodTable, LimitedAccuracy, Timings
+        # During bootstrap, Base will later include this into its own "IRShow module"
+        Compiler.include(IRShow, "ssair/show.jl")
+    end
+end
+
+end
+
+end

base/compiler/abstractinterpretation.jl renamed to Compiler/src/abstractinterpretation.jl

@@ -2047,7 +2047,7 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs
                 elsetype = rt === Const(true)  ? Bottom : widenslotwrapper(aty)
                 return Conditional(a, thentype, elsetype)
             end
-        elseif f === Core.Compiler.not_int
+        elseif f === Core.Intrinsics.not_int
             aty = argtypes[2]
             if isa(aty, Conditional)
                 thentype = rt === Const(false) ? Bottom : aty.elsetype
@@ -2119,7 +2119,7 @@ function form_partially_defined_struct(@nospecialize(obj), @nospecialize(name))
     else
         fldidx > nminfld || return nothing
     end
-    return PartialStruct(objt0, Any[obj isa PartialStruct && i≤length(obj.fields) ?
+    return PartialStruct(fallback_lattice, objt0, Any[obj isa PartialStruct && i≤length(obj.fields) ?
         obj.fields[i] : fieldtype(objt0,i) for i = 1:fldidx])
 end
 
@@ -2955,7 +2955,7 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, vtypes::Union{V
                 # - any refinement information is available (`anyrefine`), or when
                 # - `nargs` is greater than `n_initialized` derived from the struct type
                 #   information alone
-                rt = PartialStruct(rt, ats)
+                rt = PartialStruct(𝕃ᵢ, rt, ats)
             end
         else
             rt = refine_partial_type(rt)
@@ -2990,7 +2990,7 @@ function abstract_eval_splatnew(interp::AbstractInterpreter, e::Expr, vtypes::Un
                     all(i::Int -> ⊑(𝕃ᵢ, (at.fields::Vector{Any})[i], fieldtype(t, i)), 1:n)
                 end))
             nothrow = isexact
-            rt = PartialStruct(rt, at.fields::Vector{Any})
+            rt = PartialStruct(𝕃ᵢ, rt, at.fields::Vector{Any})
         end
     else
         rt = refine_partial_type(rt)
@@ -3524,7 +3524,7 @@ end
             end
             fields[i] = a
         end
-        anyrefine && return PartialStruct(rt.typ, fields)
+        anyrefine && return PartialStruct(𝕃ᵢ, rt.typ, fields)
     end
     if isa(rt, PartialOpaque)
         return rt # XXX: this case was missed in #39512

base/compiler/abstractlattice.jl renamed to Compiler/src/abstractlattice.jl

@@ -0,0 +1,66 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# make sure that typeinf is executed before turning on typeinf_ext
+# this ensures that typeinf_ext doesn't recurse before it can add the item to the workq
+# especially try to make sure any recursive and leaf functions have concrete signatures,
+# since we won't be able to specialize & infer them at runtime
+
+activate_codegen!() = ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+
+function bootstrap!()
+    let time() = ccall(:jl_clock_now, Float64, ())
+        println("Compiling the compiler. This may take several minutes ...")
+        interp = NativeInterpreter()
+
+        # analyze_escapes_tt = Tuple{typeof(analyze_escapes), IRCode, Int, TODO}
+        optimize_tt = Tuple{typeof(optimize), NativeInterpreter, OptimizationState{NativeInterpreter}, InferenceResult}
+        fs = Any[
+            # we first create caches for the optimizer, because they contain many loop constructions
+            # and they're better to not run in interpreter even during bootstrapping
+            #=analyze_escapes_tt,=# optimize_tt,
+            # then we create caches for inference entries
+            typeinf_ext, typeinf, typeinf_edge,
+        ]
+        # tfuncs can't be inferred from the inference entries above, so here we infer them manually
+        for x in T_FFUNC_VAL
+            push!(fs, x[3])
+        end
+        for i = 1:length(T_IFUNC)
+            if isassigned(T_IFUNC, i)
+                x = T_IFUNC[i]
+                push!(fs, x[3])
+            else
+                println(stderr, "WARNING: tfunc missing for ", reinterpret(IntrinsicFunction, Int32(i)))
+            end
+        end
+        starttime = time()
+        for f in fs
+            if isa(f, DataType) && f.name === typename(Tuple)
+                tt = f
+            else
+                tt = Tuple{typeof(f), Vararg{Any}}
+            end
+            for m in _methods_by_ftype(tt, 10, get_world_counter())::Vector
+                # remove any TypeVars from the intersection
+                m = m::MethodMatch
+                typ = Any[m.spec_types.parameters...]
+                for i = 1:length(typ)
+                    typ[i] = unwraptv(typ[i])
+                end
+                typeinf_type(interp, m.method, Tuple{typ...}, m.sparams)
+            end
+        end
+        endtime = time()
+        println("Base.Compiler ──── ", sub_float(endtime,starttime), " seconds")
+    end
+    activate_codegen!()
+end
+
+function activate!(; reflection=true, codegen=false)
+    if reflection
+        Base.REFLECTION_COMPILER[] = Compiler
+    end
+    if codegen
+        activate_codegen!()
+    end
+end

Compiler/src/bootstrap.jl

@@ -351,13 +351,13 @@ function decode_effects(e::UInt32)
     return Effects(
         UInt8((e >> 0) & 0x07),
         UInt8((e >> 3) & 0x03),
-        _Bool((e >> 5) & 0x01),
-        _Bool((e >> 6) & 0x01),
-        _Bool((e >> 7) & 0x01),
+        Bool((e >> 5) & 0x01),
+        Bool((e >> 6) & 0x01),
+        Bool((e >> 7) & 0x01),
         UInt8((e >> 8) & 0x03),
         UInt8((e >> 10) & 0x03),
         UInt8((e >> 12) & 0x03),
-        _Bool((e >> 14) & 0x01))
+        Bool((e >> 14) & 0x01))
 end
 
 decode_statement_effects_override(ssaflag::UInt32) =