Rewrite most gensym macros to use automatic hygiene instead (#59239)

Manual `gensym` code often contains a lot of mistakes, either because the
user uses something like `.` or `+`, or because it combines code from
multiple modules (gensym is only unique within a pre-compile unit). This
replaces most uses for macro local variables with proper scope markers.
I did not rewrite `_lift_one_interp_helper` or `replace_ref_begin_end_`
however, since, while possible by adding `esc` to every argument that
has not used a gensym value, if any other argument did is a value, I
worried that could lead to macroexpand.scm making more new mistakes so I
left if for a separate PR. Better yet, we could make a unhygienic-scope
and unescape pair for marking the inverse/dual of the usual operations
(marking a symbol as unescaped within a region of unhygienic (escaped)
code to make these various uses easier to implement.

But also do rewrite `replace_ref_begin_end_` to respect argument order
and evaluation count (similar to its julia-syntax.scm counterpart) and
scoping (not adding `let` unpredictably).

Author: vtjnash

base/Base.jl

@@ -30,6 +30,9 @@ let os = ccall(:jl_get_UNAME, Any, ())
     end
 end
 
+# metaprogramming
+include("meta.jl")
+
 # subarrays
 include("subarray.jl")
 include("views.jl")
@@ -157,9 +160,6 @@ include("weakkeydict.jl")
 # ScopedValues
 include("scopedvalues.jl")
 
-# metaprogramming
-include("meta.jl")
-
 # Logging
 include("logging/logging.jl")
 using .CoreLogging

base/boot.jl

@@ -746,7 +746,7 @@ end
 
 # module providing the IR object model
 # excluding types already exported by Core (GlobalRef, QuoteNode, Expr, LineNumberNode)
-# any type beyond these is self-quoting (see also Base.is_ast_node)
+# any type beyond these is self-quoting (see also Base.isa_ast_node)
 module IR
 
 export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,

base/cartesian.jl

@@ -36,15 +36,14 @@ If you want just a post-expression, supply [`nothing`](@ref) for the pre-express
 parentheses and semicolons, you can supply multi-statement expressions.
 """
 macro nloops(N, itersym, rangeexpr, args...)
-    _nloops(N, itersym, rangeexpr, args...)
+    _nloops(N, itersym, true, rangeexpr, args...)
 end
 
-function _nloops(N::Int, itersym::Symbol, arraysym::Symbol, args::Expr...)
-    @gensym d
-    _nloops(N, itersym, :($d->Base.axes($arraysym, $d)), args...)
+function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, arraysym::Symbol, args::Expr...)
+    _nloops(N, itersym, false, :(d->axes($(esc(arraysym)), d)), args...)
 end
 
-function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...)
+function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, rangeexpr::Expr, args::Expr...)
     if rangeexpr.head !== :->
         throw(ArgumentError("second argument must be an anonymous function expression to compute the range"))
     end
@@ -55,14 +54,16 @@ function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...)
     ex = Expr(:escape, body)
     for dim = 1:N
         itervar = inlineanonymous(itersym, dim)
+        itervar = esc(itervar)
         rng = inlineanonymous(rangeexpr, dim)
-        preexpr = length(args) > 1 ? inlineanonymous(args[1], dim) : (:(nothing))
-        postexpr = length(args) > 2 ? inlineanonymous(args[2], dim) : (:(nothing))
+        esc_rng && (rng = esc(rng))
+        preexpr = length(args) > 1 ? esc(inlineanonymous(args[1], dim)) : nothing
+        postexpr = length(args) > 2 ? esc(inlineanonymous(args[2], dim)) : nothing
         ex = quote
-            for $(esc(itervar)) = $(esc(rng))
-                $(esc(preexpr))
+            for $itervar = $rng
+                $preexpr
                 $ex
-                $(esc(postexpr))
+                $postexpr
             end
         end
     end
@@ -290,14 +291,15 @@ struct LReplace{S<:AbstractString}
 end
 LReplace(sym::Symbol, val::Integer) = LReplace(sym, string(sym), val)
 
-lreplace(ex::Expr, sym::Symbol, val) = lreplace!(copy(ex), LReplace(sym, val))
+lreplace(ex::Expr, sym::Symbol, val) = lreplace!(copy(ex), LReplace(sym, val), false, 0)
 
-function lreplace!(sym::Symbol, r::LReplace)
+function lreplace!(sym::Symbol, r::LReplace, in_quote_context::Bool, escs::Int)
+    escs == 0 || return sym
     sym == r.pat_sym && return r.val
-    Symbol(lreplace!(string(sym), r))
+    Symbol(lreplace_string!(string(sym), r))
 end
 
-function lreplace!(str::AbstractString, r::LReplace)
+function lreplace_string!(str::String, r::LReplace)
     i = firstindex(str)
     pat = r.pat_str
     j = firstindex(pat)
@@ -329,7 +331,7 @@ function lreplace!(str::AbstractString, r::LReplace)
         if matching && j > lastindex(pat)
             if i > lastindex(str) || str[i] == '_'
                 # We have a match
-                return string(str[1:prevind(str, istart)], r.val, lreplace!(str[i:end], r))
+                return string(str[1:prevind(str, istart)], r.val, lreplace_string!(str[i:end], r))
             end
             matching = false
             j = firstindex(pat)
@@ -339,24 +341,42 @@ function lreplace!(str::AbstractString, r::LReplace)
     str
 end
 
-function lreplace!(ex::Expr, r::LReplace)
+function lreplace!(ex::Expr, r::LReplace, in_quote_context::Bool, escs::Int)
     # Curly-brace notation, which acts like parentheses
-    if ex.head === :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]::Symbol), "_")
-        excurly = exprresolve(lreplace!(ex.args[2], r))
+    if !in_quote_context && ex.head === :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]::Symbol), "_")
+        excurly = exprresolve(lreplace!(ex.args[2], r, in_quote_context, escs))
         if isa(excurly, Int)
             return Symbol(ex.args[1]::Symbol, excurly)
         else
             ex.args[2] = excurly
             return ex
         end
+    elseif ex.head === :meta || ex.head === :inert
+        return ex
+    elseif ex.head === :$
+        # no longer an executable expression (handle all equivalent forms of :inert, :quote, and QuoteNode the same way)
+        in_quote_context = false
+    elseif ex.head === :quote
+        # executable again
+        in_quote_context = true
+    elseif ex.head === :var"hygienic-scope"
+        # no longer our expression
+        escs += 1
+    elseif ex.head === :escape
+        # our expression again once zero
+        escs == 0 && return ex
+        escs -= 1
+    elseif ex.head === :macrocall
+        # n.b. blithely go about altering arguments to macros also, assuming that is at all what the user intended
+        # it is probably the user's fault if they put a macro inside here and didn't mean for it to get rewritten
     end
     for i in 1:length(ex.args)
-        ex.args[i] = lreplace!(ex.args[i], r)
+        ex.args[i] = lreplace!(ex.args[i], r, in_quote_context, escs)
     end
     ex
 end
 
-lreplace!(arg, r::LReplace) = arg
+lreplace!(@nospecialize(arg), r::LReplace, in_quote_context::Bool, escs::Int) = arg
 
 
 poplinenum(arg) = arg

base/experimental.jl

@@ -41,7 +41,7 @@ within this scope, even if the compiler can't prove this to be the case.
     Experimental API. Subject to change without deprecation.
 """
 macro aliasscope(body)
-    sym = gensym()
+    sym = :aliasscope_result
     quote
         $(Expr(:aliasscope))
         $sym = $(esc(body))

base/meta.jl

@@ -18,9 +18,121 @@ export quot,
 
 public parse
 
-using Base: isidentifier, isoperator, isunaryoperator, isbinaryoperator, ispostfixoperator
 import Base: isexpr
 
+## AST decoding helpers ##
+
+is_id_start_char(c::AbstractChar) = ccall(:jl_id_start_char, Cint, (UInt32,), c) != 0
+is_id_char(c::AbstractChar) = ccall(:jl_id_char, Cint, (UInt32,), c) != 0
+
+"""
+     isidentifier(s) -> Bool
+
+Return whether the symbol or string `s` contains characters that are parsed as
+a valid ordinary identifier (not a binary/unary operator) in Julia code;
+see also [`Base.isoperator`](@ref).
+
+Internally Julia allows any sequence of characters in a `Symbol` (except `\\0`s),
+and macros automatically use variable names containing `#` in order to avoid
+naming collision with the surrounding code. In order for the parser to
+recognize a variable, it uses a limited set of characters (greatly extended by
+Unicode). `isidentifier()` makes it possible to query the parser directly
+whether a symbol contains valid characters.
+
+# Examples
+```jldoctest
+julia> Meta.isidentifier(:x), Meta.isidentifier("1x")
+(true, false)
+```
+"""
+function isidentifier(s::AbstractString)
+    x = Iterators.peel(s)
+    isnothing(x) && return false
+    (s == "true" || s == "false") && return false
+    c, rest = x
+    is_id_start_char(c) || return false
+    return all(is_id_char, rest)
+end
+isidentifier(s::Symbol) = isidentifier(string(s))
+
+is_op_suffix_char(c::AbstractChar) = ccall(:jl_op_suffix_char, Cint, (UInt32,), c) != 0
+
+_isoperator(s) = ccall(:jl_is_operator, Cint, (Cstring,), s) != 0
+
+"""
+    isoperator(s::Symbol)
+
+Return `true` if the symbol can be used as an operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.isoperator(:+), Meta.isoperator(:f)
+(true, false)
+```
+"""
+isoperator(s::Union{Symbol,AbstractString}) = _isoperator(s) || ispostfixoperator(s)
+
+"""
+    isunaryoperator(s::Symbol)
+
+Return `true` if the symbol can be used as a unary (prefix) operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.isunaryoperator(:-), Meta.isunaryoperator(:√), Meta.isunaryoperator(:f)
+(true, true, false)
+```
+"""
+isunaryoperator(s::Symbol) = ccall(:jl_is_unary_operator, Cint, (Cstring,), s) != 0
+is_unary_and_binary_operator(s::Symbol) = ccall(:jl_is_unary_and_binary_operator, Cint, (Cstring,), s) != 0
+is_syntactic_operator(s::Symbol) = ccall(:jl_is_syntactic_operator, Cint, (Cstring,), s) != 0
+
+"""
+    isbinaryoperator(s::Symbol)
+
+Return `true` if the symbol can be used as a binary (infix) operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.isbinaryoperator(:-), Meta.isbinaryoperator(:√), Meta.isbinaryoperator(:f)
+(true, false, false)
+```
+"""
+function isbinaryoperator(s::Symbol)
+    return _isoperator(s) && (!isunaryoperator(s) || is_unary_and_binary_operator(s)) &&
+        s !== Symbol("'")
+end
+
+"""
+    ispostfixoperator(s::Union{Symbol,AbstractString})
+
+Return `true` if the symbol can be used as a postfix operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.ispostfixoperator(Symbol("'")), Meta.ispostfixoperator(Symbol("'ᵀ")), Meta.ispostfixoperator(:-)
+(true, true, false)
+```
+"""
+function ispostfixoperator(s::Union{Symbol,AbstractString})
+    s = String(s)::String
+    return startswith(s, '\'') && all(is_op_suffix_char, SubString(s, 2))
+end
+
+const keyword_syms = IdSet{Symbol}([
+    :baremodule, :begin, :break, :catch, :const, :continue, :do, :else, :elseif,
+    :end, :export, :var"false", :finally, :for, :function, :global, :if, :import,
+    :let, :local, :macro, :module, :public, :quote, :return, :struct, :var"true",
+    :try, :using, :while ])
+
+function is_valid_identifier(sym)
+    return (isidentifier(sym) && !(sym in keyword_syms)) ||
+        (_isoperator(sym) &&
+        !(sym in (Symbol("'"), :(::), :?)) &&
+        !is_syntactic_operator(sym)
+    )
+end
+
 """
     Meta.quot(ex)::Expr
 
@@ -516,6 +628,21 @@ function unescape(@nospecialize ex)
     return ex
 end
 
+"""
+    Meta.reescape(unescaped_expr, original_expr)
+
+Re-wrap `unescaped_expr` with the same level of escaping as `original_expr` had.
+This is the inverse operation of [`unescape`](@ref) - if the original expression
+was escaped, the unescaped expression is wrapped in `:escape` again.
+"""
+function reescape(@nospecialize(unescaped_expr), @nospecialize(original_expr))
+    if isexpr(original_expr, :escape) || isexpr(original_expr, :var"hygienic-scope")
+        return reescape(Expr(:escape, unescaped_expr), original_expr.args[1])
+    else
+        return unescaped_expr
+    end
+end
+
 """
     Meta.uncurly(expr)