Expand import/using to newer Base runtime calls (#43)

Calls to `import` and `using` are expanded by lowering as of the
changes in JuliaLang/julia#57965
and no longer dealt with by the C function `jl_toplevel_eval_flex`.

This implies we can't use `eval()` for these if we want to activate
JuliaLowering in Core, or we'll hit a stack overflow.

I've chosen to duplicate the flisp lowering here for consistency and
import paths are thus lowered to a restricted kind of quoted `Expr`.
(It's mildly annoying to rely on quoted `Expr` in the lowered paths than
the previous use of `Core.svec` but deleting the svec representation
allows us to use `Base._eval_import` and `Base._eval_using` directly so
seems like a worthy simplification.)

Similarly, use a precomputed vector of names in public/export
expansion - this list can be computed at expansion time rather
than emitting each element into the lowered code individually.

Includes minor test+CI fixes julia 1.12 in support of JETLS.

Author: c42f

.github/workflows/CI.yml

@@ -19,6 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
+          - '~1.12.0-rc1'
           - 'nightly'
         os:
           - ubuntu-latest

src/desugaring.jl

@@ -4098,7 +4098,9 @@ end
 #-------------------------------------------------------------------------------
 # Expand import / using / export
 
-function _append_importpath(ctx, path_spec, path)
+function expand_importpath(path)
+    @chk kind(path) == K"importpath"
+    path_spec = Expr(:.)
     prev_was_dot = true
     for component in children(path)
         k = kind(component)
@@ -4114,13 +4116,12 @@ function _append_importpath(ctx, path_spec, path)
             throw(LoweringError(component, "invalid import path: `.` in identifier path"))
         end
         prev_was_dot = is_dot
-        push!(path_spec, @ast(ctx, component, name::K"String"))
+        push!(path_spec.args, Symbol(name))
     end
-    path_spec
+    return path_spec
 end
 
-function expand_import(ctx, ex)
-    is_using = kind(ex) == K"using"
+function expand_import_or_using(ctx, ex)
     if kind(ex[1]) == K":"
         # import M: x.y as z, w
         # (import (: (importpath M) (as (importpath x y) z) (importpath w)))
@@ -4131,57 +4132,87 @@ function expand_import(ctx, ex)
         #  (call core.svec  2 "x" "y" "z"  1 "w" "w"))
         @chk numchildren(ex[1]) >= 2
         from = ex[1][1]
-        @chk kind(from) == K"importpath"
-        from_path = @ast ctx from [K"call"
-            "svec"::K"core"
-            _append_importpath(ctx, SyntaxList(ctx), from)...
-        ]
+        from_path = @ast ctx from QuoteNode(expand_importpath(from))::K"Value"
         paths = ex[1][2:end]
     else
         # import A.B
         # (using (importpath A B))
-        # (call module_import true nothing (call core.svec 1 "w"))
+        # (call eval_import true nothing (call core.svec 1 "w"))
         @chk numchildren(ex) >= 1
-        from_path = nothing_(ctx, ex)
+        from_path = nothing
         paths = children(ex)
     end
-    path_spec = SyntaxList(ctx)
-    for path in paths
+    # Here we represent the paths as quoted `Expr` data structures
+    path_specs = SyntaxList(ctx)
+    for spec in paths
         as_name = nothing
-        if kind(path) == K"as"
-            @chk numchildren(path) == 2
-            as_name = path[2]
-            @chk kind(as_name) == K"Identifier"
-            path = path[1]
+        if kind(spec) == K"as"
+            @chk numchildren(spec) == 2
+            @chk kind(spec[2]) == K"Identifier"
+            as_name = Symbol(spec[2].name_val)
+            path = QuoteNode(Expr(:as, expand_importpath(spec[1]), as_name))
+        else
+            path = QuoteNode(expand_importpath(spec))
         end
-        @chk kind(path) == K"importpath"
-        push!(path_spec, @ast(ctx, path, numchildren(path)::K"Integer"))
-        _append_importpath(ctx, path_spec, path)
-        push!(path_spec, isnothing(as_name) ? nothing_(ctx, ex) :
-                         @ast(ctx, as_name, as_name.name_val::K"String"))
+        push!(path_specs, @ast ctx spec path::K"Value")
     end
-    @ast ctx ex [K"block"
-        [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]]
-        [K"call"
-            module_import ::K"Value"
+    is_using = kind(ex) == K"using"
+    stmts = SyntaxList(ctx)
+    if isnothing(from_path)
+        for spec in path_specs
+            if is_using
+                push!(stmts,
+                    @ast ctx spec [K"call"
+                        eval_using   ::K"Value"
+                        ctx.mod      ::K"Value"
+                        spec
+                    ]
+                )
+            else
+                push!(stmts,
+                    @ast ctx spec [K"call"
+                        eval_import   ::K"Value"
+                        (!is_using)   ::K"Bool"
+                        ctx.mod       ::K"Value"
+                        "nothing"     ::K"top"
+                        spec
+                    ]
+                )
+            end
+            # latestworld required between imports so that previous symbols
+            # become visible
+            push!(stmts, @ast ctx spec (::K"latestworld"))
+        end
+    else
+        push!(stmts, @ast ctx ex [K"call"
+            eval_import   ::K"Value"
+            (!is_using)   ::K"Bool"
             ctx.mod       ::K"Value"
-            is_using      ::K"Value"
             from_path
-            [K"call"
-                "svec"::K"core"
-                path_spec...
-            ]
-        ]
+            path_specs...
+        ])
+        push!(stmts, @ast ctx ex (::K"latestworld"))
+    end
+    @ast ctx ex [K"block"
+        [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]]
+        stmts...
+        [K"removable" "nothing"::K"core"]
     ]
 end
 
 # Expand `public` or `export`
 function expand_public(ctx, ex)
+    identifiers = String[]
+    for e in children(ex)
+        @chk kind(e) == K"Identifier" (ex, "Expected identifier")
+        push!(identifiers, e.name_val)
+    end
+    (e.name_val::K"String" for e in children(ex))
     @ast ctx ex [K"call"
-        module_public::K"Value"
+        eval_public::K"Value"
         ctx.mod::K"Value"
         (kind(ex) == K"export")::K"Bool"
-        (e.name_val::K"String" for e in children(ex))...
+        identifiers::K"Value"
     ]
 end
 
@@ -4421,7 +4452,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
     elseif k == K"module"
         expand_module(ctx, ex)
     elseif k == K"import" || k == K"using"
-        expand_import(ctx, ex)
+        expand_import_or_using(ctx, ex)
     elseif k == K"export" || k == K"public"
         expand_public(ctx, ex)
     elseif k == K"abstract" || k == K"primitive"

src/linear_ir.jl

@@ -855,6 +855,9 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
         end
         ctx.is_toplevel_thunk && emit_latestworld(ctx, ex)
     elseif k == K"latestworld"
+        if needs_value
+            throw(LoweringError(ex, "misplaced latestsworld"))
+        end
         emit_latestworld(ctx, ex)
     elseif k == K"latestworld_if_toplevel"
         ctx.is_toplevel_thunk && emit_latestworld(ctx, ex)

src/runtime.jl

@@ -110,7 +110,7 @@ end
 
 #--------------------------------------------------
 # Functions called by closure conversion
-function eval_closure_type(mod, closure_type_name, field_names, field_is_box)
+function eval_closure_type(mod::Module, closure_type_name::Symbol, field_names, field_is_box)
     type_params = Core.TypeVar[]
     field_types = []
     for (name, isbox) in zip(field_names, field_is_box)
@@ -129,7 +129,7 @@ function eval_closure_type(mod, closure_type_name, field_names, field_is_box)
                             false,
                             length(field_names))
     Core._setsuper!(type, Core.Function)
-    Base.eval(mod, :(const $closure_type_name = $type))
+    @ccall jl_set_const(mod::Module, closure_type_name::Symbol, type::Any)::Cvoid
     Core._typebody!(false, type, Core.svec(field_types...))
     type
 end
@@ -176,39 +176,29 @@ function eval_module(parentmod, modname, body)
     ))
 end
 
-# Evaluate content of `import` or `using` statement
-function module_import(into_mod::Module, is_using::Bool,
-                       from_mod::Union{Nothing,Core.SimpleVector}, paths::Core.SimpleVector)
-    # For now, this function converts our lowered representation back to Expr
-    # and calls eval() to avoid replicating all of the fiddly logic in
-    # jl_toplevel_eval_flex.
-    # TODO: ccall Julia runtime functions directly?
-    #   * jl_module_using jl_module_use_as
-    #   * import_module jl_module_import_as
-    path_args = []
-    i = 1
-    while i < length(paths)
-        nsyms = paths[i]::Int
-        n = i + nsyms
-        path = Expr(:., [Symbol(paths[i+j]::String) for j = 1:nsyms]...)
-        as_name = paths[i+nsyms+1]
-        push!(path_args, isnothing(as_name) ? path :
-                         Expr(:as, path, Symbol(as_name)))
-        i += nsyms + 2
+const _Base_has_eval_import = isdefined(Base, :_eval_import)
+
+function eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...)
+    if _Base_has_eval_import
+        Base._eval_import(imported, to, from, paths...)
+    else
+        head = imported ? :import : :using
+        ex = isnothing(from) ?
+            Expr(head, paths...) :
+            Expr(head, Expr(Symbol(":"), from, paths...))
+        Base.eval(to, ex)
     end
-    ex = if isnothing(from_mod)
-        Expr(is_using ? :using : :import,
-             path_args...)
+end
+
+function eval_using(to::Module, path::Expr)
+    if _Base_has_eval_import
+        Base._eval_using(to, path)
     else
-        from_path = Expr(:., [Symbol(s::String) for s in from_mod]...)
-        Expr(is_using ? :using : :import,
-             Expr(:(:), from_path, path_args...))
+        Base.eval(to, Expr(:using, path))
     end
-    eval(into_mod, ex)
-    nothing
 end
 
-function module_public(mod::Module, is_exported::Bool, identifiers...)
+function eval_public(mod::Module, is_exported::Bool, identifiers)
     # symbol jl_module_public is no longer exported as of #57765
     eval(mod, Expr((is_exported ? :export : :public), map(Symbol, identifiers)...))
 end

test/hooks.jl

@@ -19,20 +19,22 @@ const JL = JuliaLowering
         end
     end
 
-    @testset "integration: `JuliaLowering.activate!`" begin
-        prog = parseall(Expr, "global asdf = 1")
-        JL.activate!()
-        out = Core.eval(test_mod, prog)
-        JL.activate!(false)
-        @test out === 1
-        @test isdefined(test_mod, :asdf)
+    if isdefined(Core, :_lower)
+        @testset "integration: `JuliaLowering.activate!`" begin
+            prog = parseall(Expr, "global asdf = 1")
+            JL.activate!()
+            out = Core.eval(test_mod, prog)
+            JL.activate!(false)
+            @test out === 1
+            @test isdefined(test_mod, :asdf)
 
-        prog = parseall(Expr, "module M; x = 1; end")
-        JL.activate!()
-        out = Core.eval(test_mod, prog)
-        JL.activate!(false)
-        @test out isa Module
-        @test isdefined(test_mod, :M)
-        @test isdefined(test_mod.M, :x)
+            prog = parseall(Expr, "module M; x = 1; end")
+            JL.activate!()
+            out = Core.eval(test_mod, prog)
+            JL.activate!(false)
+            @test out isa Module
+            @test isdefined(test_mod, :M)
+            @test isdefined(test_mod.M, :x)
+        end
     end
 end

test/import.jl

@@ -39,4 +39,37 @@ end
 """)
 @test C.D.f === C.E.f
 
+# Test that `using` F brings in the symbol G immediately
+F = JuliaLowering.include_string(test_mod, """
+module F
+    export G
+    module G
+        export G_global
+        G_global = "exported from G"
+    end
+end
+""")
+JuliaLowering.include_string(test_mod, """
+using .F, .G
+""")
+@test test_mod.F === F
+@test test_mod.G === F.G
+@test test_mod.G_global === "exported from G"
+
+# Similarly, that import makes symbols available immediately
+H = JuliaLowering.include_string(test_mod, """
+module H
+    module I
+        module J
+        end
+    end
+end
+""")
+JuliaLowering.include_string(test_mod, """
+import .H.I, .I.J
+""")
+@test test_mod.I === H.I
+@test test_mod.J === H.I.J
+@test test_mod.G_global === "exported from G"
+
 end