work around CSTParser type piracy

Author: MikeInnes

Manifest.toml

@@ -3,6 +3,12 @@
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[CSTParser]]
+deps = ["Tokenize"]
+git-tree-sha1 = "376a39f1862000442011390f1edf5e7f4dcc7142"
+uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
+version = "0.6.0"
+
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
 git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f"

Project.toml

@@ -2,6 +2,7 @@ name = "MacroTools"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 
 [deps]
+CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/MacroTools.jl

@@ -18,6 +18,8 @@ include("examples/forward.jl")
 
 include("cstparser/CSTParser.jl")
 
+module hack; using CSTParser; end
+
 using .CSTParser
 
 include("patch/diff.jl")

src/cstparser/components/operators.jl

@@ -43,10 +43,6 @@ precedence(x::AbstractToken) = precedence(x.kind)
 precedence(x::OPERATOR) = precedence(x.kind)
 
 
-isoperator(kind) = Tokens.begin_ops < kind < Tokens.end_ops
-isoperator(t::AbstractToken) = isoperator(t.kind)
-
-
 isunaryop(op) = false
 isunaryop(op::OPERATOR) = isunaryop(op.kind)
 isunaryop(t::AbstractToken) = isunaryop(t.kind)
@@ -243,7 +239,7 @@ end
 # Parse power (special case for preceding unary ops)
 function parse_operator_power(ps::ParseState, @nospecialize(ret), op)
     nextarg = @precedence ps PowerOp - LtoR(PowerOp) @closer ps inwhere parse_expression(ps)
-    
+
     if ret isa UnaryOpCall
         nextarg = BinaryOpCall(ret.arg, op, nextarg)
         ret = UnaryOpCall(ret.op, nextarg)
@@ -257,7 +253,7 @@ end
 # parse where
 function parse_operator_where(ps::ParseState, @nospecialize(ret), op)
     nextarg = @precedence ps LazyAndOp @closer ps inwhere parse_expression(ps)
-    
+
     if nextarg isa EXPR{Braces}
         args = nextarg.args
     else
@@ -286,7 +282,7 @@ function parse_operator_dot(ps::ParseState, @nospecialize(ret), op)
         if ps.nt.kind == Tokens.LPAREN
             nextarg = @closeparen ps @precedence ps DotOp - LtoR(DotOp) parse_expression(ps)
             nextarg = EXPR{Quote}(Any[op2, nextarg])
-        else    
+        else
             nextarg = @precedence ps DotOp - LtoR(DotOp) parse_unary(ps, op2)
         end
     elseif ps.nt.kind == Tokens.EX_OR && ps.nnt.kind == Tokens.LPAREN
@@ -312,7 +308,7 @@ end
 
 function parse_operator_anon_func(ps::ParseState, @nospecialize(ret), op)
     arg = @closer ps comma @precedence ps 0 parse_expression(ps)
-    
+
     if !(arg isa EXPR{Begin} || (arg isa EXPR{InvisBrackets} && arg.args[2] isa EXPR{Block}))
         arg = EXPR{Block}(Any[arg])
     end
@@ -352,7 +348,7 @@ function parse_operator(ps::ParseState, @nospecialize(ret), op)
     else
         ltor = K == Tokens.LPIPE ? true : LtoR(P)
         nextarg = @precedence ps P - ltor parse_expression(ps)
-        
+
         if issyntaxcall(op)
             ret = BinarySyntaxOpCall(ret, op, nextarg)
         else

src/cstparser/utils.jl

@@ -9,7 +9,7 @@ function closer(ps::ParseState)
         ps.nt.kind == Tokens.LBRACE ||
         ps.nt.kind == Tokens.LSQUARE ||
         (ps.nt.kind == Tokens.STRING && isemptyws(ps.ws)) ||
-        ((ps.nt.kind == Tokens.RPAREN || ps.nt.kind == Tokens.RSQUARE) && isidentifier(ps.nt))  
+        ((ps.nt.kind == Tokens.RPAREN || ps.nt.kind == Tokens.RSQUARE) && isidentifier(ps.nt))
     )) ||
     (ps.nt.kind == Tokens.COMMA && ps.closer.precedence > 0) ||
     ps.nt.kind == Tokens.ENDMARKER ||
@@ -20,10 +20,10 @@ function closer(ps::ParseState)
     (ps.closer.paren && ps.nt.kind == Tokens.RPAREN) ||
     (ps.closer.brace && ps.nt.kind == Tokens.RBRACE) ||
     (ps.closer.square && ps.nt.kind == Tokens.RSQUARE) ||
-    ps.nt.kind == Tokens.ELSEIF || 
+    ps.nt.kind == Tokens.ELSEIF ||
     ps.nt.kind == Tokens.ELSE ||
-    ps.nt.kind == Tokens.CATCH || 
-    ps.nt.kind == Tokens.FINALLY || 
+    ps.nt.kind == Tokens.CATCH ||
+    ps.nt.kind == Tokens.FINALLY ||
     (ps.closer.ifop && isoperator(ps.nt) && (precedence(ps.nt) <= 0 || ps.nt.kind == Tokens.COLON)) ||
     (ps.closer.range && (ps.nt.kind == Tokens.FOR || iscomma(ps.nt) || ps.nt.kind == Tokens.IF)) ||
     (ps.closer.ws && !isemptyws(ps.ws) &&
@@ -32,7 +32,7 @@ function closer(ps::ParseState)
         !(!ps.closer.inmacro && ps.nt.kind == Tokens.FOR) &&
         !(ps.nt.kind == Tokens.DO) &&
         !(
-            (isbinaryop(ps.nt) && !(isemptyws(ps.nws) && isunaryop(ps.nt) && ps.closer.wsop)) || 
+            (isbinaryop(ps.nt) && !(isemptyws(ps.nws) && isunaryop(ps.nt) && ps.closer.wsop)) ||
             (isunaryop(ps.t) && ps.ws.kind == WS)
         )) ||
     (ps.nt.startbyte ≥ ps.closer.stop) ||
@@ -209,8 +209,6 @@ end
 
 isidentifier(t::AbstractToken) = t.kind == Tokens.IDENTIFIER
 
-isliteral(t::AbstractToken) = Tokens.begin_literal < t.kind < Tokens.end_literal
-
 isbool(t::AbstractToken) =  Tokens.TRUE ≤ t.kind ≤ Tokens.FALSE
 iscomma(t::AbstractToken) =  t.kind == Tokens.COMMA
 
@@ -224,7 +222,7 @@ isinstance(t::AbstractToken) = isidentifier(t) ||
 
 ispunctuation(t::AbstractToken) = t.kind == Tokens.COMMA ||
                           t.kind == Tokens.END ||
-                          Tokens.LSQUARE ≤ t.kind ≤ Tokens.RPAREN || 
+                          Tokens.LSQUARE ≤ t.kind ≤ Tokens.RPAREN ||
                           t.kind == Tokens.AT_SIGN
 
 isstring(x) = false
@@ -235,7 +233,7 @@ is_float(x) = x isa LITERAL && x.kind == Tokens.FLOAT
 is_number(x) = x isa LITERAL && (x.kind == Tokens.INTEGER || x.kind == Tokens.FLOAT)
 is_nothing(x) = x isa LITERAL && x.kind == Tokens.NOTHING
 
-isajuxtaposition(ps::ParseState, ret) = ((is_number(ret) && (ps.nt.kind == Tokens.IDENTIFIER || ps.nt.kind == Tokens.LPAREN || ps.nt.kind == Tokens.CMD || ps.nt.kind == Tokens.STRING || ps.nt.kind == Tokens.TRIPLE_STRING)) || 
+isajuxtaposition(ps::ParseState, ret) = ((is_number(ret) && (ps.nt.kind == Tokens.IDENTIFIER || ps.nt.kind == Tokens.LPAREN || ps.nt.kind == Tokens.CMD || ps.nt.kind == Tokens.STRING || ps.nt.kind == Tokens.TRIPLE_STRING)) ||
         ((ret isa UnarySyntaxOpCall && is_prime(ret.arg2) && ps.nt.kind == Tokens.IDENTIFIER) ||
         ((ps.t.kind == Tokens.RPAREN || ps.t.kind == Tokens.RSQUARE) && (ps.nt.kind == Tokens.IDENTIFIER || ps.nt.kind == Tokens.CMD)) ||
         ((ps.t.kind == Tokens.STRING || ps.t.kind == Tokens.TRIPLE_STRING) && (ps.nt.kind == Tokens.STRING || ps.nt.kind == Tokens.TRIPLE_STRING)))) || ((ps.t.kind in (Tokens.INTEGER, Tokens.FLOAT) || ps.t.kind in (Tokens.RPAREN,Tokens.RSQUARE,Tokens.RBRACE)) && ps.nt.kind == Tokens.IDENTIFIER)
@@ -386,7 +384,7 @@ function check_file(file, ret, neq)
     str = read(file, String)
     x0, cstfailed, sp = cst_parsefile(str)
     x1, flispfailed = flisp_parsefile(str)
-    
+
     print("\r                             ")
     if !isempty(sp)
         printstyled(file, color = :blue)
@@ -408,7 +406,7 @@ function check_file(file, ret, neq)
         printstyled(string("    ", c1), bold = true, color = :light_green)
         println()
         push!(ret, (file, :noteq))
-    end    
+    end
 end
 
 function check_base(dir = dirname(Base.find_source_file("essentials.jl")), display = false)
@@ -428,7 +426,7 @@ function check_base(dir = dirname(Base.find_source_file("essentials.jl")), displ
                 N += 1
                 try
                     print("\r", rpad(string(N), 5), rpad(string(round(fail / N * 100, sigdigits = 3)), 8), rpad(string(round(err / N * 100, sigdigits = 3)), 8), rpad(string(round(neq / N * 100, sigdigits = 3)), 8))
-                    
+
                     check_file(file, ret, neq)
                 catch er
                     isa(er, InterruptException) && rethrow(er)
@@ -495,15 +493,15 @@ of its components. Returns a vector of failing expressions.
 function check_span(x::EXPR{StringH}, neq = []) end
 function check_span(x::T, neq = []) where T <: Union{IDENTIFIER,LITERAL,OPERATOR,KEYWORD,PUNCTUATION} neq end
 
-function check_span(x::UnaryOpCall, neq = []) 
+function check_span(x::UnaryOpCall, neq = [])
     check_span(x.op)
     check_span(x.arg)
     if x.op.fullspan + x.arg.fullspan != x.fullspan
         push!(neq, x)
     end
     neq
 end
-function check_span(x::UnarySyntaxOpCall, neq = []) 
+function check_span(x::UnarySyntaxOpCall, neq = [])
     check_span(x.arg1)
     check_span(x.arg2)
     if x.arg1.fullspan + x.arg2.fullspan != x.fullspan
@@ -619,7 +617,7 @@ Base.iterate(x::BinaryOpCall, s) = s > 2 ? nothing : (getfield(x, s + 1), s + 1)
 Base.length(x::BinaryOpCall) = 3
 
 Base.iterate(x::WhereOpCall) = x.arg1, 1
-function Base.iterate(x::WhereOpCall, s) 
+function Base.iterate(x::WhereOpCall, s)
     if s == 1
         return x.op, 2
     elseif s < length(x)
@@ -703,5 +701,3 @@ function _unescape_string(io, s::AbstractString)
         end
     end
 end
-
-