use Compiler stdlib if available (#613)

This requires JuliaLang/julia#56553 to be merged, and the new
Compiler.jl stdlib to be registered in General beforehand.

Author: aviatesk

.github/workflows/CI.yml

@@ -16,7 +16,7 @@ jobs:
           - version: '1' # current stable
             os: ubuntu-latest
             arch: x64
-          - version: '1.10.0' # lowerest version supported
+          - version: '1.10' # lowest version supported
             os: ubuntu-latest
             arch: x64
           - version: '1.12-nightly' # next release

Project.toml

@@ -1,10 +1,11 @@
 name = "Cthulhu"
 uuid = "f68482b8-f384-11e8-15f7-abe071a5a75f"
+version = "2.16.0"
 authors = ["Valentin Churavy <v.churavy@gmail.com> and contributors"]
-version = "2.15.3"
 
 [deps]
 CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
+Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1"
 FoldingTrees = "1eca21be-9b9b-4ed8-839a-6d8ae26b1781"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
@@ -18,6 +19,7 @@ WidthLimitedIO = "b8c1c048-cf81-46c6-9da0-18c1d99e41f2"
 
 [compat]
 CodeTracking = "0.5, 1"
+Compiler = "0.0.2"
 FoldingTrees = "1"
 InteractiveUtils = "1.9"
 JuliaSyntax = "0.4"
@@ -28,7 +30,7 @@ TypedSyntax = "1.3.0"
 UUIDs = "1.9"
 Unicode = "1.9"
 WidthLimitedIO = "1"
-julia = "1.10.0"
+julia = "1.10"
 
 [extras]
 DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6"

src/Cthulhu.jl

@@ -12,9 +12,21 @@ using TypedSyntax
 using WidthLimitedIO
 
 using Core: MethodInstance, MethodMatch
-const CC = Core.Compiler
-using .CC: Effects, EFFECTS_TOTAL, LimitedAccuracy,
-    compileable_specialization, ignorelimited, specialize_method
+@static if VERSION ≥ v"1.12.0-DEV.1581"
+    using Compiler: Compiler as CC
+    using Compiler.IRShow: IRShow
+else
+    const CC = Core.Compiler
+    const IRShow = Base.IRShow
+end
+using Core.IR
+using .CC: AbstractInterpreter, ApplyCallInfo, CallInfo as CCCallInfo, ConstCallInfo,
+    EFFECTS_TOTAL, Effects, IncrementalCompact, InferenceParams, InferenceResult,
+    InferenceState, IRCode, LimitedAccuracy, MethodMatchInfo, MethodResultPure,
+    NativeInterpreter, NoCallInfo, OptimizationParams, OptimizationState,
+    UnionSplitApplyCallInfo, UnionSplitInfo, WorldRange, WorldView,
+    argextype, argtypes_to_type, compileable_specialization, ignorelimited, singleton_type,
+    specialize_method, sptypes_from_meth_instance, widenconst
 using Base: @constprop, default_tt, isvarargtype, unwrapva, unwrap_unionall, rewrap_unionall
 const mapany = Base.mapany
 
@@ -803,7 +815,7 @@ function _descend(term::AbstractTerminal, interp::AbstractInterpreter, curs::Abs
             @static if VERSION < v"1.12.0-DEV.669"
                 view_cmd(iostream, mi, optimize, debuginfo, world, CONFIG)
             else
-                src = Core.Compiler.typeinf_code(interp, mi, true)
+                src = CC.typeinf_code(interp, mi, true)
                 view_cmd(iostream, mi, src, optimize, debuginfo, world, CONFIG)
             end
             display_CI = false

src/codeview.jl

@@ -100,17 +100,15 @@ function cthulhu_ast(io::IO, mi, ::Bool, debuginfo, ::UInt, config::CthulhuConfi
     end
 end
 
-using Base.IRShow: IRShow, _stmt, _type, should_print_ssa_type, IRShowConfig, show_ir
-
 const __debuginfo = merge(IRShow.__debuginfo, Dict(
-    :compact => src -> src isa CodeInfo ? __debuginfo[:source](src)
-                                        : IRShow.inline_linfo_printer(src)
-))
+    :compact => function (src)
+        src isa CodeInfo ? IRShow.__debuginfo[:source](src) : IRShow.inline_linfo_printer(src)
+    end))
 
 function is_type_unstable(code::Union{IRCode, CodeInfo}, idx::Int, used::BitSet)
-    stmt = _stmt(code, idx)
-    type = _type(code, idx)
-    should_print_ssa_type(stmt) || return false
+    stmt = IRShow._stmt(code, idx)
+    type = IRShow._type(code, idx)
+    IRShow.should_print_ssa_type(stmt) || return false
     # `used` only contains used SSA values and ignores slots
     in_use = in(idx, used) || Meta.isexpr(stmt, :(=))
     return in_use && is_type_unstable(type)
@@ -308,7 +306,7 @@ function cthulhu_typed(io::IO, debuginfo::Symbol,
     should_print_stmt = hide_type_stable ? is_type_unstable : Returns(true)
     bb_color = (src isa IRCode && debuginfo === :compact) ? :normal : :light_black
 
-    irshow_config = IRShowConfig(preprinter, postprinter; should_print_stmt, bb_color)
+    irshow_config = IRShow.IRShowConfig(preprinter, postprinter; should_print_stmt, bb_color)
 
     if !inline_cost && iswarn
         print(lambda_io, "Body")
@@ -319,14 +317,14 @@ function cthulhu_typed(io::IO, debuginfo::Symbol,
         println(lambda_io)
     else
         isa(mi, MethodInstance) || throw("`mi::MethodInstance` is required")
-        cfg = src isa IRCode ? src.cfg : Core.Compiler.compute_basic_blocks(src.code)
+        cfg = src isa IRCode ? src.cfg : CC.compute_basic_blocks(src.code)
         max_bb_idx_size = length(string(length(cfg.blocks)))
         str = irshow_config.line_info_preprinter(lambda_io, " "^(max_bb_idx_size + 2), -1)
         callsite = Callsite(0, MICallInfo(mi, rettype, effects, exct), :invoke)
         println(lambda_io, "∘ ", "─"^(max_bb_idx_size), str, " ", callsite)
     end
 
-    show_ir(lambda_io, src, irshow_config)
+    IRShow.show_ir(lambda_io, src, irshow_config)
     return nothing
 end
 
@@ -547,7 +545,7 @@ else
         raw = false,
         config = CONFIG,
     )
-        src = Core.Compiler.typeinf_code(b.interp, b.mi, true)
+        src = CC.typeinf_code(b.interp, b.mi, true)
         return cthulhu_llvm(
             io,
             b.mi,
@@ -570,7 +568,7 @@ else
         raw = false,
         config = CONFIG,
     )
-        src = Core.Compiler.typeinf_code(b.interp, b.mi, true)
+        src = CC.typeinf_code(b.interp, b.mi, true)
         return cthulhu_native(
             io,
             b.mi,

src/interpreter.jl

@@ -1,7 +1,3 @@
-using .CC: AbstractInterpreter, CallInfo as CCCallInfo, CodeInfo, CodeInstance,
-    InferenceParams, InferenceResult, InferenceState, IRCode, NativeInterpreter,
-    NoCallInfo, OptimizationParams, OptimizationState, SSAValue, WorldRange, WorldView
-
 struct InferredSource
     src::CodeInfo
     stmt_info::Vector{CCCallInfo}

src/reflection.jl

@@ -3,11 +3,7 @@
 ##
 
 using Base.Meta
-import .CC: widenconst, argextype, Const, MethodMatchInfo,
-    UnionSplitApplyCallInfo, UnionSplitInfo, ConstCallInfo,
-    MethodResultPure, ApplyCallInfo,
-    sptypes_from_meth_instance, argtypes_to_type
-import Base: may_invoke_generator
+using Base: may_invoke_generator
 
 transform(::Val, callsite) = callsite
 function transform(::Val{:CuFunction}, callsite, callexpr, CI, mi, slottypes; world=get_world_counter())
@@ -269,8 +265,8 @@ function is_call_expr(x::Expr, optimize::Bool)
 end
 
 function dce!(ir::IRCode)
-    ir = Core.Compiler.compact!(ir, #=allow_cfg_transform=#true)
-    ir = Core.Compiler.compact!(ir, #=allow_cfg_transform=#true)
+    ir = CC.compact!(ir, #=allow_cfg_transform=#true)
+    ir = CC.compact!(ir, #=allow_cfg_transform=#true)
     return ir
 end
 
@@ -355,14 +351,14 @@ end
 
 function add_sourceline!(locs::Vector{Tuple{Core.LineInfoNode,Int}}, src::Union{CodeInfo,IRCode}, stmtidx::Int, caller::MethodInstance)
     @static if VERSION ≥ v"1.12.0-DEV.173"
-    stack = Base.IRShow.buildLineInfoNode(src.debuginfo, caller, stmtidx)
+    stack = IRShow.buildLineInfoNode(src.debuginfo, caller, stmtidx)
     for (i, di) in enumerate(stack)
         loc = Core.LineInfoNode(Main, di.method, di.file, di.line, zero(Int32))
         push!(locs, (loc, i-1))
     end
     else # VERSION < v"1.12.0-DEV.173"
     if isa(src, IRCode)
-        stack = Base.IRShow.compute_loc_stack(src.linetable, src.stmts.line[stmtidx])
+        stack = IRShow.compute_loc_stack(src.linetable, src.stmts.line[stmtidx])
         for (i, idx) in enumerate(stack)
             line = src.linetable[idx]
             line.line == 0 && continue

test/irutils.jl

@@ -1,10 +1,10 @@
-using Core: CodeInfo, ReturnNode, MethodInstance
-using Core.Compiler: IRCode, IncrementalCompact, singleton_type
+using Core.IR
+using Cthulhu: Cthulhu
 using Base.Meta: isexpr
 using InteractiveUtils: gen_call_with_extracted_types_and_kwargs
 
-argextype(@nospecialize args...) = Core.Compiler.argextype(args...)
-argextype(@nospecialize(x), src::CodeInfo) = argextype(x, src, Core.Compiler.VarState[])
+argextype(@nospecialize args...) = Cthulhu.CC.argextype(args...)
+argextype(@nospecialize(x), src::CodeInfo) = argextype(x, src, Cthulhu.CC.VarState[])
 code_typed1(args...; kwargs...) = first(only(code_typed(args...; kwargs...)))::CodeInfo
 macro code_typed1(ex0...)
     return gen_call_with_extracted_types_and_kwargs(__module__, :code_typed1, ex0)
@@ -20,9 +20,9 @@ isreturn(@nospecialize x) = isa(x, ReturnNode)
 
 # check if `x` is a dynamic call of a given function
 iscall(y) = @nospecialize(x) -> iscall(y, x)
-function iscall((src, f)::Tuple{IR,Base.Callable}, @nospecialize(x)) where IR<:Union{CodeInfo,IRCode,IncrementalCompact}
+function iscall((src, f), @nospecialize(x))
     return iscall(x) do @nospecialize x
-        singleton_type(argextype(x, src)) === f
+        Cthulhu.CC.singleton_type(argextype(x, src)) === f
     end
 end
 function iscall(pred::Base.Callable, @nospecialize(x))

test/setup.jl

@@ -3,8 +3,12 @@ if isdefined(parentmodule(@__MODULE__), :VSCodeServer)
     using ..VSCodeServer
 end
 
+@static if VERSION ≥ v"1.12.0-DEV.1581"
+InteractiveUtils.@activate Compiler # use the Compiler.jl stdlib for the Base reflections too
+end
+
 function cthulhu_info(@nospecialize(f), @nospecialize(tt=());
-                      optimize=true, interp=Core.Compiler.NativeInterpreter())
+                      optimize=true, interp=Cthulhu.CC.NativeInterpreter())
     (interp, mi) = Cthulhu.mkinterp(f, tt; interp)
     (; src, rt, exct, infos, slottypes, effects) =
         Cthulhu.lookup(interp, mi, optimize; allow_no_src=true)

test/test_AbstractInterpreter.jl

@@ -5,34 +5,65 @@ if isdefined(parentmodule(@__MODULE__), :VSCodeServer)
     using ..VSCodeServer
 end
 
-const CC = Core.Compiler
+@doc """
+    @newinterp NewInterpreter [ephemeral_cache::Bool=false]
+
+Defines new `NewInterpreter <: AbstractInterpreter` whose cache is separated
+from the native code cache, satisfying the minimum interface requirements.
 
+When the `ephemeral_cache=true` option is specified, `NewInterpreter` will hold
+`CodeInstance` in an ephemeral non-integrated cache, rather than in the integrated
+`Core.Compiler.InternalCodeCache`.
+Keep in mind that ephemeral cache lacks support for invalidation and doesn't persist across
+sessions. However it is an usual Julia object of the type `code_cache::IdDict{MethodInstance,CodeInstance}`,
+making it easier for debugging and inspecting the compiler behavior.
+"""
 @static if VERSION ≥ v"1.11.0-DEV.1552"
-macro newinterp(InterpName)
-    InterpCacheName = QuoteNode(Symbol(string(InterpName, "Cache")))
+macro newinterp(InterpName, ephemeral_cache::Bool=false)
+    cache_token = QuoteNode(gensym(string(InterpName, "CacheToken")))
+    InterpCacheName = esc(Symbol(string(InterpName, "Cache")))
     InterpName = esc(InterpName)
     C = Core
-    CC = Core.Compiler
+    CC = Cthulhu.CC
     quote
+        $(ephemeral_cache && quote
+        struct $InterpCacheName
+            dict::IdDict{$C.MethodInstance,$C.CodeInstance}
+        end
+        $InterpCacheName() = $InterpCacheName(IdDict{$C.MethodInstance,$C.CodeInstance}())
+        end)
         struct $InterpName <: $CC.AbstractInterpreter
             meta # additional information
             world::UInt
             inf_params::$CC.InferenceParams
             opt_params::$CC.OptimizationParams
             inf_cache::Vector{$CC.InferenceResult}
+            $(ephemeral_cache && :(code_cache::$InterpCacheName))
             function $InterpName(meta = nothing;
-                                 world::UInt = Base.get_world_counter(),
-                                 inf_params::$CC.InferenceParams = $CC.InferenceParams(),
-                                 opt_params::$CC.OptimizationParams = $CC.OptimizationParams(),
-                                 inf_cache::Vector{$CC.InferenceResult} = $CC.InferenceResult[])
-                return new(meta, world, inf_params, opt_params, inf_cache)
+                                    world::UInt = Base.get_world_counter(),
+                                    inf_params::$CC.InferenceParams = $CC.InferenceParams(),
+                                    opt_params::$CC.OptimizationParams = $CC.OptimizationParams(),
+                                    inf_cache::Vector{$CC.InferenceResult} = $CC.InferenceResult[],
+                                    $(ephemeral_cache ?
+                                    Expr(:kw, :(code_cache::$InterpCacheName), :($InterpCacheName())) :
+                                    Expr(:kw, :_, :nothing)))
+                return $(ephemeral_cache ?
+                    :(new(meta, world, inf_params, opt_params, inf_cache, code_cache)) :
+                    :(new(meta, world, inf_params, opt_params, inf_cache)))
             end
         end
         $CC.InferenceParams(interp::$InterpName) = interp.inf_params
         $CC.OptimizationParams(interp::$InterpName) = interp.opt_params
         $CC.get_inference_world(interp::$InterpName) = interp.world
         $CC.get_inference_cache(interp::$InterpName) = interp.inf_cache
-        $CC.cache_owner(::$InterpName) = $InterpCacheName
+        $CC.cache_owner(::$InterpName) = $cache_token
+        $(ephemeral_cache && quote
+        $CC.code_cache(interp::$InterpName) = $CC.WorldView(interp.code_cache, $CC.WorldRange(interp.world))
+        $CC.get(wvc::$CC.WorldView{$InterpCacheName}, mi::$C.MethodInstance, default) = get(wvc.cache.dict, mi, default)
+        $CC.getindex(wvc::$CC.WorldView{$InterpCacheName}, mi::$C.MethodInstance) = getindex(wvc.cache.dict, mi)
+        $CC.haskey(wvc::$CC.WorldView{$InterpCacheName}, mi::$C.MethodInstance) = haskey(wvc.cache.dict, mi)
+        $CC.setindex!(wvc::$CC.WorldView{$InterpCacheName}, ci::$C.CodeInstance, mi::$C.MethodInstance) = setindex!(wvc.cache.dict, ci, mi)
+        end)
     end
 end
 else
@@ -79,16 +110,11 @@ macro newinterp(InterpName)
 end
 end # if VERSION ≥ v"1.11.0-DEV.1552"
 
-@doc """
-@newinterp NewInterpreter
-
-Defines new `NewInterpreter <: AbstractInterpreter` whose cache is separated
-from the native code cache, satisfying the minimum interface requirements.
-""" var"@newinterp"
+const CC = Cthulhu.CC
 
 # `OverlayMethodTable`
 # --------------------
-import Base.Experimental: @MethodTable, @overlay
+using Base.Experimental: @MethodTable, @overlay
 
 @newinterp MTOverlayInterp
 @MethodTable OverlayedMT