Rewrite macros to use automatic hygiene

Manual `gensym` code often contains a lot of mistakes, either because
the user uses something like `.` or `+`, or because it combines code
from multiple modules (gensym is only unique within a pre-compile unit).
This replaces most uses for macro local variables with proper scope
markers, and checks for those markers correctly when doing replacements.
I did not fully rewrite `_lift_one_interp_helper` or
`replace_ref_begin_end_` however, since, while possible by adding `esc`
to every argument that has not used a gensym value, if any other
argument did is a value, I worried that could lead to macroexpand.scm
making more new mistakes so I left if for a separate PR. Better yet, we
could make a unhygienic-scope and unescape pair for marking the
inverse/dual of the usual operations (marking a symbol as unescaped
within a region of unhygienic (escaped) code to make these various uses
easier to implement.

But rewrite `replace_ref_begin_end_` to respect argument order and
evaluation count (similar to its julia-syntax.scm counterpart) and
scoping (not adding `let` unpredictably). This required also rewriting
`gen_call_with_extracted_types` to properly support the ways that
might require rewriting the expression, particularly against the new
`::` syntax support in the interactive macros.

Author: vtjnash

base/boot.jl

@@ -746,7 +746,7 @@ end
 
 # module providing the IR object model
 # excluding types already exported by Core (GlobalRef, QuoteNode, Expr, LineNumberNode)
-# any type beyond these is self-quoting (see also Base.is_ast_node)
+# any type beyond these is self-quoting (see also Base.isa_ast_node)
 module IR
 
 export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,

base/cartesian.jl

@@ -36,15 +36,14 @@ If you want just a post-expression, supply [`nothing`](@ref) for the pre-express
 parentheses and semicolons, you can supply multi-statement expressions.
 """
 macro nloops(N, itersym, rangeexpr, args...)
-    _nloops(N, itersym, rangeexpr, args...)
+    _nloops(N, itersym, true, rangeexpr, args...)
 end
 
-function _nloops(N::Int, itersym::Symbol, arraysym::Symbol, args::Expr...)
-    @gensym d
-    _nloops(N, itersym, :($d->Base.axes($arraysym, $d)), args...)
+function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, arraysym::Symbol, args::Expr...)
+    _nloops(N, itersym, false, :(d->axes($(esc(arraysym)), d)), args...)
 end
 
-function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...)
+function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, rangeexpr::Expr, args::Expr...)
     if rangeexpr.head !== :->
         throw(ArgumentError("second argument must be an anonymous function expression to compute the range"))
     end
@@ -55,14 +54,16 @@ function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...)
     ex = Expr(:escape, body)
     for dim = 1:N
         itervar = inlineanonymous(itersym, dim)
+        itervar = esc(itervar)
         rng = inlineanonymous(rangeexpr, dim)
-        preexpr = length(args) > 1 ? inlineanonymous(args[1], dim) : (:(nothing))
-        postexpr = length(args) > 2 ? inlineanonymous(args[2], dim) : (:(nothing))
+        esc_rng && (rng = esc(rng))
+        preexpr = length(args) > 1 ? esc(inlineanonymous(args[1], dim)) : nothing
+        postexpr = length(args) > 2 ? esc(inlineanonymous(args[2], dim)) : nothing
         ex = quote
-            for $(esc(itervar)) = $(esc(rng))
-                $(esc(preexpr))
+            for $itervar = $rng
+                $preexpr
                 $ex
-                $(esc(postexpr))
+                $postexpr
             end
         end
     end
@@ -290,14 +291,15 @@ struct LReplace{S<:AbstractString}
 end
 LReplace(sym::Symbol, val::Integer) = LReplace(sym, string(sym), val)
 
-lreplace(ex::Expr, sym::Symbol, val) = lreplace!(copy(ex), LReplace(sym, val))
+lreplace(ex::Expr, sym::Symbol, val) = lreplace!(copy(ex), LReplace(sym, val), false, 0)
 
-function lreplace!(sym::Symbol, r::LReplace)
+function lreplace!(sym::Symbol, r::LReplace, in_quote_context::Bool, escs::Int)
+    escs == 0 || return sym
     sym == r.pat_sym && return r.val
-    Symbol(lreplace!(string(sym), r))
+    Symbol(lreplace_string!(string(sym), r))
 end
 
-function lreplace!(str::AbstractString, r::LReplace)
+function lreplace_string!(str::String, r::LReplace)
     i = firstindex(str)
     pat = r.pat_str
     j = firstindex(pat)
@@ -329,7 +331,7 @@ function lreplace!(str::AbstractString, r::LReplace)
         if matching && j > lastindex(pat)
             if i > lastindex(str) || str[i] == '_'
                 # We have a match
-                return string(str[1:prevind(str, istart)], r.val, lreplace!(str[i:end], r))
+                return string(str[1:prevind(str, istart)], r.val, lreplace_string!(str[i:end], r))
             end
             matching = false
             j = firstindex(pat)
@@ -339,24 +341,42 @@ function lreplace!(str::AbstractString, r::LReplace)
     str
 end
 
-function lreplace!(ex::Expr, r::LReplace)
+function lreplace!(ex::Expr, r::LReplace, in_quote_context::Bool, escs::Int)
     # Curly-brace notation, which acts like parentheses
-    if ex.head === :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]::Symbol), "_")
-        excurly = exprresolve(lreplace!(ex.args[2], r))
+    if !in_quote_context && ex.head === :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]::Symbol), "_")
+        excurly = exprresolve(lreplace!(ex.args[2], r, in_quote_context, escs))
         if isa(excurly, Int)
             return Symbol(ex.args[1]::Symbol, excurly)
         else
             ex.args[2] = excurly
             return ex
         end
+    elseif ex.head === :meta || ex.head === :inert
+        return ex
+    elseif ex.head === :$
+        # no longer an executable expression (handle all equivalent forms of :inert, :quote, and QuoteNode the same way)
+        in_quote_context = false
+    elseif ex.head === :quote
+        # executable again
+        in_quote_context = true
+    elseif ex.head === :var"hygienic-scope"
+        # no longer our expression
+        escs += 1
+    elseif ex.head === :escape
+        # our expression again once zero
+        escs == 0 && return ex
+        escs -= 1
+    elseif ex.head === :macrocall
+        # n.b. blithely go about altering arguments to macros also, assuming that is at all what the user intended
+        # it is probably the user's fault if they put a macro inside here and didn't mean for it to get rewritten
     end
     for i in 1:length(ex.args)
-        ex.args[i] = lreplace!(ex.args[i], r)
+        ex.args[i] = lreplace!(ex.args[i], r, in_quote_context, escs)
     end
     ex
 end
 
-lreplace!(arg, r::LReplace) = arg
+lreplace!(@nospecialize(arg), r::LReplace, in_quote_context::Bool, escs::Int) = arg
 
 
 poplinenum(arg) = arg

base/experimental.jl

@@ -41,7 +41,7 @@ within this scope, even if the compiler can't prove this to be the case.
     Experimental API. Subject to change without deprecation.
 """
 macro aliasscope(body)
-    sym = gensym()
+    sym = :aliasscope_result
     quote
         $(Expr(:aliasscope))
         $sym = $(esc(body))

base/meta.jl

@@ -628,6 +628,21 @@ function unescape(@nospecialize ex)
     return ex
 end
 
+"""
+    Meta.reescape(unescaped_expr, original_expr)
+
+Re-wrap `unescaped_expr` with the same level of escaping as `original_expr` had.
+This is the inverse operation of [`unescape`](@ref) - if the original expression
+was escaped, the unescaped expression is wrapped in `:escape` again.
+"""
+function reescape(@nospecialize(unescaped_expr), @nospecialize(original_expr))
+    if isexpr(original_expr, :escape) || isexpr(original_expr, :var"hygienic-scope")
+        return reescape(Expr(:escape, unescaped_expr), original_expr.args[1])
+    else
+        return unescaped_expr
+    end
+end
+
 """
     Meta.uncurly(expr)
 

base/reflection.jl

@@ -1305,19 +1305,19 @@ macro invoke(ex)
     f, args, kwargs = destructure_callex(topmod, ex)
     types = Expr(:curly, :Tuple)
     out = Expr(:call, GlobalRef(Core, :invoke))
-    isempty(kwargs) || push!(out.args, Expr(:parameters, kwargs...))
-    push!(out.args, f)
+    isempty(kwargs) || push!(out.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...))
+    push!(out.args, esc(f))
     push!(out.args, types)
     for arg in args
         if isexpr(arg, :(::))
-            push!(out.args, arg.args[1])
-            push!(types.args, arg.args[2])
+            push!(out.args, esc(arg.args[1]))
+            push!(types.args, esc(arg.args[2]))
         else
-            push!(out.args, arg)
-            push!(types.args, Expr(:call, GlobalRef(Core, :Typeof), arg))
+            push!(out.args, esc(arg))
+            push!(types.args, Expr(:call, GlobalRef(Core, :Typeof), esc(arg)))
         end
     end
-    return esc(out)
+    return out
 end
 
 getglobalref(gr::GlobalRef, world::UInt) = ccall(:jl_eval_globalref, Any, (Any, UInt), gr, world)
@@ -1367,42 +1367,42 @@ macro invokelatest(ex)
 
     if !isa(f, GlobalRef)
         out_f = Expr(:call, GlobalRef(Base, :invokelatest))
-        isempty(kwargs) || push!(out_f.args, Expr(:parameters, kwargs...))
+        isempty(kwargs) || push!(out_f.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...))
 
         if isexpr(f, :(.))
-            s = gensym()
+            s = :s
             check = quote
-                $s = $(f.args[1])
+                $s = $(esc(f.args[1]))
                 isa($s, Module)
             end
-            push!(out_f.args, Expr(:(.), s, f.args[2]))
+            push!(out_f.args, Expr(:(.), s, esc(f.args[2])))
         else
-            push!(out_f.args, f)
+            push!(out_f.args, esc(f))
         end
-        append!(out_f.args, args)
+        append!(out_f.args, Any[esc(arg) for arg in args])
 
         if @isdefined(s)
-            f = :(GlobalRef($s, $(f.args[2])))
-        elseif !isa(f, Symbol)
-            return esc(out_f)
+            f = :(GlobalRef($s, $(esc(f.args[2]))))
+        elseif isa(f, Symbol)
+            check = esc(:($(Expr(:isglobal, f))))
         else
-            check = :($(Expr(:isglobal, f)))
+            return out_f
         end
     end
 
     out_gr = Expr(:call, GlobalRef(Base, :invokelatest_gr))
-    isempty(kwargs) || push!(out_gr.args, Expr(:parameters, kwargs...))
+    isempty(kwargs) || push!(out_gr.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...))
     push!(out_gr.args, isa(f, GlobalRef) ? QuoteNode(f) :
                        isa(f, Symbol) ? QuoteNode(GlobalRef(__module__, f)) :
                        f)
-    append!(out_gr.args, args)
+    append!(out_gr.args, Any[esc(arg) for arg in args])
 
     if isa(f, GlobalRef)
-        return esc(out_gr)
+        return out_gr
     end
 
     # f::Symbol
-    return esc(:($check ? $out_gr : $out_f))
+    return :($check ? $out_gr : $out_f)
 end
 
 function destructure_callex(topmod::Module, @nospecialize(ex))

base/show.jl

@@ -1,8 +1,9 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
 using .Compiler: has_typevar
-using .Meta: isidentifier, isoperator, isunaryoperator, isbinaryoperator, ispostfixoperator, 
-            is_id_start_char, is_id_char, _isoperator, is_syntactic_operator, is_valid_identifier
+using .Meta: isidentifier, isoperator, isunaryoperator, isbinaryoperator, ispostfixoperator,
+            is_id_start_char, is_id_char, _isoperator, is_syntactic_operator, is_valid_identifier,
+            is_unary_and_binary_operator
 
 function show(io::IO, ::MIME"text/plain", u::UndefInitializer)
     show(io, u)

base/simdloop.jl

@@ -60,22 +60,22 @@ function compile(x, ivdep)
     check_body!(x)
 
     var,range = parse_iteration_space(x.args[1])
-    r = gensym("r") # Range value
-    j = gensym("i") # Iteration variable for outer loop
-    n = gensym("n") # Trip count for inner loop
-    i = gensym("i") # Trip index for inner loop
-    quote
+    # r: Range value
+    # j: Iteration variable for outer loop
+    # n: Trip count for inner loop
+    # i: Trip index for inner loop
+    return quote
         # Evaluate range value once, to enhance type and data flow analysis by optimizers.
-        let $r = $range
-            for $j in Base.simd_outer_range($r)
-                let $n = Base.simd_inner_length($r,$j)
-                    if zero($n) < $n
+        let r = $(esc(range))
+            for j in Base.simd_outer_range(r)
+                let n = Base.simd_inner_length(r,j)
+                    if zero(n) < n
                         # Lower loop in way that seems to work best for LLVM 3.3 vectorizer.
-                        let $i = zero($n)
-                            while $i < $n
-                                local $var = Base.simd_index($r,$j,$i)
-                                $(x.args[2])        # Body of loop
-                                $i += 1
+                        let i = zero(n)
+                            while i < n
+                                local $(esc(var)) = Base.simd_index(r,j,i)
+                                $(esc(x.args[2]))        # Body of loop
+                                i += 1
                                 $(Expr(:loopinfo, Symbol("julia.simdloop"), ivdep))  # Mark loop as SIMD loop
                             end
                         end
@@ -125,12 +125,12 @@ either case, your inner loop should have the following properties to allow vecto
 * No iteration ever waits on a previous iteration to make forward progress.
 """
 macro simd(forloop)
-    esc(compile(forloop, nothing))
+    compile(forloop, nothing)
 end
 
 macro simd(ivdep, forloop)
     if ivdep === :ivdep
-        esc(compile(forloop, Symbol("julia.ivdep")))
+        compile(forloop, Symbol("julia.ivdep"))
     else
         throw(SimdError("Only ivdep is valid as the first argument to @simd"))
     end

base/some.jl

@@ -152,8 +152,9 @@ macro something(args...)
     which is why we need the last argument first
     when building the final expression.
     =#
-    for arg in reverse(args)
-        val = gensym()
+    for i in reverse(eachindex(args))
+        arg  = args[i]
+        val = Cartesian.inlineanonymous(:val, i)
         expr = quote
             $val = $(esc(arg))
             if !isnothing($val)