Rewrite most gensym macros to use automatic hygiene instead

base/Base.jl

@@ -30,6 +30,9 @@ let os = ccall(:jl_get_UNAME, Any, ())
     end
 end
 
+# metaprogramming
+include("meta.jl")
+
 # subarrays
 include("subarray.jl")
 include("views.jl")
@@ -157,9 +160,6 @@ include("weakkeydict.jl")
 # ScopedValues
 include("scopedvalues.jl")
 
-# metaprogramming
-include("meta.jl")
-
 # Logging
 include("logging/logging.jl")
 using .CoreLogging

base/boot.jl

@@ -746,7 +746,7 @@ end
 
 # module providing the IR object model
 # excluding types already exported by Core (GlobalRef, QuoteNode, Expr, LineNumberNode)
-# any type beyond these is self-quoting (see also Base.is_ast_node)
+# any type beyond these is self-quoting (see also Base.isa_ast_node)
 module IR
 
 export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,

base/cartesian.jl

@@ -36,15 +36,14 @@ If you want just a post-expression, supply [`nothing`](@ref) for the pre-express
 parentheses and semicolons, you can supply multi-statement expressions.
 """
 macro nloops(N, itersym, rangeexpr, args...)
-    _nloops(N, itersym, rangeexpr, args...)
+    _nloops(N, itersym, true, rangeexpr, args...)
 end
 
-function _nloops(N::Int, itersym::Symbol, arraysym::Symbol, args::Expr...)
-    @gensym d
-    _nloops(N, itersym, :($d->Base.axes($arraysym, $d)), args...)
+function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, arraysym::Symbol, args::Expr...)
+    _nloops(N, itersym, false, :(d->axes($(esc(arraysym)), d)), args...)
 end
 
-function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...)
+function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, rangeexpr::Expr, args::Expr...)
     if rangeexpr.head !== :->
         throw(ArgumentError("second argument must be an anonymous function expression to compute the range"))
     end
@@ -55,11 +54,13 @@ function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...)
     ex = Expr(:escape, body)
     for dim = 1:N
         itervar = inlineanonymous(itersym, dim)
+        itervar = esc(itervar)
         rng = inlineanonymous(rangeexpr, dim)
+        esc_rng && (rng = esc(rng))
         preexpr = length(args) > 1 ? inlineanonymous(args[1], dim) : (:(nothing))
         postexpr = length(args) > 2 ? inlineanonymous(args[2], dim) : (:(nothing))
         ex = quote
-            for $(esc(itervar)) = $(esc(rng))
+            for $itervar = $rng
                 $(esc(preexpr))
                 $ex
                 $(esc(postexpr))

base/experimental.jl

@@ -41,7 +41,7 @@ within this scope, even if the compiler can't prove this to be the case.
     Experimental API. Subject to change without deprecation.
 """
 macro aliasscope(body)
-    sym = gensym()
+    sym = :aliasscope_result
     quote
         $(Expr(:aliasscope))
         $sym = $(esc(body))

base/meta.jl

@@ -18,9 +18,121 @@ export quot,
 
 public parse
 
-using Base: isidentifier, isoperator, isunaryoperator, isbinaryoperator, ispostfixoperator
 import Base: isexpr
 
+## AST decoding helpers ##
+
+is_id_start_char(c::AbstractChar) = ccall(:jl_id_start_char, Cint, (UInt32,), c) != 0
+is_id_char(c::AbstractChar) = ccall(:jl_id_char, Cint, (UInt32,), c) != 0
+
+"""
+     isidentifier(s) -> Bool
+
+Return whether the symbol or string `s` contains characters that are parsed as
+a valid ordinary identifier (not a binary/unary operator) in Julia code;
+see also [`Base.isoperator`](@ref).
+
+Internally Julia allows any sequence of characters in a `Symbol` (except `\\0`s),
+and macros automatically use variable names containing `#` in order to avoid
+naming collision with the surrounding code. In order for the parser to
+recognize a variable, it uses a limited set of characters (greatly extended by
+Unicode). `isidentifier()` makes it possible to query the parser directly
+whether a symbol contains valid characters.
+
+# Examples
+```jldoctest
+julia> Meta.isidentifier(:x), Meta.isidentifier("1x")
+(true, false)
+```
+"""
+function isidentifier(s::AbstractString)
+    x = Iterators.peel(s)
+    isnothing(x) && return false
+    (s == "true" || s == "false") && return false
+    c, rest = x
+    is_id_start_char(c) || return false
+    return all(is_id_char, rest)
+end
+isidentifier(s::Symbol) = isidentifier(string(s))
+
+is_op_suffix_char(c::AbstractChar) = ccall(:jl_op_suffix_char, Cint, (UInt32,), c) != 0
+
+_isoperator(s) = ccall(:jl_is_operator, Cint, (Cstring,), s) != 0
+
+"""
+    isoperator(s::Symbol)
+
+Return `true` if the symbol can be used as an operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.isoperator(:+), Meta.isoperator(:f)
+(true, false)
+```
+"""
+isoperator(s::Union{Symbol,AbstractString}) = _isoperator(s) || ispostfixoperator(s)
+
+"""
+    isunaryoperator(s::Symbol)
+
+Return `true` if the symbol can be used as a unary (prefix) operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.isunaryoperator(:-), Meta.isunaryoperator(:√), Meta.isunaryoperator(:f)
+(true, true, false)
+```
+"""
+isunaryoperator(s::Symbol) = ccall(:jl_is_unary_operator, Cint, (Cstring,), s) != 0
+is_unary_and_binary_operator(s::Symbol) = ccall(:jl_is_unary_and_binary_operator, Cint, (Cstring,), s) != 0
+is_syntactic_operator(s::Symbol) = ccall(:jl_is_syntactic_operator, Cint, (Cstring,), s) != 0
+
+"""
+    isbinaryoperator(s::Symbol)
+
+Return `true` if the symbol can be used as a binary (infix) operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.isbinaryoperator(:-), Meta.isbinaryoperator(:√), Meta.isbinaryoperator(:f)
+(true, false, false)
+```
+"""
+function isbinaryoperator(s::Symbol)
+    return _isoperator(s) && (!isunaryoperator(s) || is_unary_and_binary_operator(s)) &&
+        s !== Symbol("'")
+end
+
+"""
+    ispostfixoperator(s::Union{Symbol,AbstractString})
+
+Return `true` if the symbol can be used as a postfix operator, `false` otherwise.
+
+# Examples
+```jldoctest
+julia> Meta.ispostfixoperator(Symbol("'")), Meta.ispostfixoperator(Symbol("'ᵀ")), Meta.ispostfixoperator(:-)
+(true, true, false)
+```
+"""
+function ispostfixoperator(s::Union{Symbol,AbstractString})
+    s = String(s)::String
+    return startswith(s, '\'') && all(is_op_suffix_char, SubString(s, 2))
+end
+
+const keyword_syms = IdSet{Symbol}([
+    :baremodule, :begin, :break, :catch, :const, :continue, :do, :else, :elseif,
+    :end, :export, :var"false", :finally, :for, :function, :global, :if, :import,
+    :let, :local, :macro, :module, :public, :quote, :return, :struct, :var"true",
+    :try, :using, :while ])
+
+function is_valid_identifier(sym)
+    return (isidentifier(sym) && !(sym in keyword_syms)) ||
+        (_isoperator(sym) &&
+        !(sym in (Symbol("'"), :(::), :?)) &&
+        !is_syntactic_operator(sym)
+    )
+end
+
 """
     Meta.quot(ex)::Expr
 
@@ -516,6 +628,21 @@ function unescape(@nospecialize ex)
     return ex
 end
 
+"""
+    Meta.reescape(unescaped_expr, original_expr)
+
+Re-wrap `unescaped_expr` with the same level of escaping as `original_expr` had.
+This is the inverse operation of [`unescape`](@ref) - if the original expression
+was escaped, the unescaped expression is wrapped in `:escape` again.
+"""
+function reescape(@nospecialize(unescaped_expr), @nospecialize(original_expr))
+    if isexpr(original_expr, :escape) || isexpr(original_expr, :var"hygienic-scope")
+        return reescape(Expr(:escape, unescaped_expr), original_expr.args[1])
+    else
+        return unescaped_expr
+    end
+end
+
 """
     Meta.uncurly(expr)
 

base/reflection.jl

@@ -1305,19 +1305,19 @@ macro invoke(ex)
     f, args, kwargs = destructure_callex(topmod, ex)
     types = Expr(:curly, :Tuple)
     out = Expr(:call, GlobalRef(Core, :invoke))
-    isempty(kwargs) || push!(out.args, Expr(:parameters, kwargs...))
-    push!(out.args, f)
+    isempty(kwargs) || push!(out.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...))
+    push!(out.args, esc(f))
     push!(out.args, types)
     for arg in args
         if isexpr(arg, :(::))
-            push!(out.args, arg.args[1])
-            push!(types.args, arg.args[2])
+            push!(out.args, esc(arg.args[1]))
+            push!(types.args, esc(arg.args[2]))
         else
-            push!(out.args, arg)
-            push!(types.args, Expr(:call, GlobalRef(Core, :Typeof), arg))
+            push!(out.args, esc(arg))
+            push!(types.args, Expr(:call, GlobalRef(Core, :Typeof), esc(arg)))
         end
     end
-    return esc(out)
+    return out
 end
 
 getglobalref(gr::GlobalRef, world::UInt) = ccall(:jl_eval_globalref, Any, (Any, UInt), gr, world)
@@ -1367,42 +1367,42 @@ macro invokelatest(ex)
 
     if !isa(f, GlobalRef)
         out_f = Expr(:call, GlobalRef(Base, :invokelatest))
-        isempty(kwargs) || push!(out_f.args, Expr(:parameters, kwargs...))
+        isempty(kwargs) || push!(out_f.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...))
 
         if isexpr(f, :(.))
-            s = gensym()
+            s = :s
             check = quote
-                $s = $(f.args[1])
+                $s = $(esc(f.args[1]))
                 isa($s, Module)
             end
-            push!(out_f.args, Expr(:(.), s, f.args[2]))
+            push!(out_f.args, Expr(:(.), s, esc(f.args[2])))
         else
-            push!(out_f.args, f)
+            push!(out_f.args, esc(f))
         end
-        append!(out_f.args, args)
+        append!(out_f.args, Any[esc(arg) for arg in args])
 
         if @isdefined(s)
-            f = :(GlobalRef($s, $(f.args[2])))
-        elseif !isa(f, Symbol)
-            return esc(out_f)
+            f = :(GlobalRef($s, $(esc(f.args[2]))))
+        elseif isa(f, Symbol)
+            check = esc(:($(Expr(:isglobal, f))))
         else
-            check = :($(Expr(:isglobal, f)))
+            return out_f
         end
     end
 
     out_gr = Expr(:call, GlobalRef(Base, :invokelatest_gr))
-    isempty(kwargs) || push!(out_gr.args, Expr(:parameters, kwargs...))
+    isempty(kwargs) || push!(out_gr.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...))
     push!(out_gr.args, isa(f, GlobalRef) ? QuoteNode(f) :
                        isa(f, Symbol) ? QuoteNode(GlobalRef(__module__, f)) :
                        f)
-    append!(out_gr.args, args)
+    append!(out_gr.args, Any[esc(arg) for arg in args])
 
     if isa(f, GlobalRef)
-        return esc(out_gr)
+        return out_gr
     end
 
     # f::Symbol
-    return esc(:($check ? $out_gr : $out_f))
+    return :($check ? $out_gr : $out_f)
 end
 
 function destructure_callex(topmod::Module, @nospecialize(ex))