diff --git a/src/JuliaLowering.jl b/src/JuliaLowering.jl index 717a3bbe..b24c5687 100644 --- a/src/JuliaLowering.jl +++ b/src/JuliaLowering.jl @@ -32,6 +32,7 @@ _include("runtime.jl") _include("syntax_macros.jl") _include("eval.jl") +_include("compat.jl") function __init__() _register_kinds() diff --git a/src/ast.jl b/src/ast.jl index 1b718e14..16239bf7 100644 --- a/src/ast.jl +++ b/src/ast.jl @@ -498,7 +498,7 @@ end # the middle of a pass. const CompileHints = Base.ImmutableDict{Symbol,Any} -function setmeta(ex::SyntaxTree; kws...) +function setmeta!(ex::SyntaxTree; kws...) @assert length(kws) == 1 # todo relax later ? key = first(keys(kws)) value = first(values(kws)) @@ -506,9 +506,12 @@ function setmeta(ex::SyntaxTree; kws...) m = get(ex, :meta, nothing) isnothing(m) ? CompileHints(key, value) : CompileHints(m, key, value) end - setattr(ex; meta=meta) + setattr!(ex; meta=meta) + ex end +setmeta(ex::SyntaxTree; kws...) = setmeta!(copy_node(ex); kws...) + function getmeta(ex::SyntaxTree, name::Symbol, default) meta = get(ex, :meta, nothing) isnothing(meta) ? default : get(meta, name, default) diff --git a/src/compat.jl b/src/compat.jl new file mode 100644 index 00000000..76dad46c --- /dev/null +++ b/src/compat.jl @@ -0,0 +1,510 @@ +const JS = JuliaSyntax + +function _insert_tree_node(graph::SyntaxGraph, k::Kind, src::SourceAttrType, + flags::UInt16=0x0000; attrs...) + id = newnode!(graph) + sethead!(graph, id, k) + flags !== 0 && setflags!(graph, id, flags) + setattr!(graph, id; source=src, attrs...) + return id +end + +""" +An Expr -> SyntaxTree transformation that should preserve semantics, but will +have low-quality provenance info (namely, each tree node will be associated with +the last seen LineNumberNode in the pre-order expr traversal). + +Last-resort option so that, for example, we can lower the output of old +Expr-producing macros. Always prefer re-parsing source text over using this. + +Supports parsed and/or macro-expanded exprs, but not lowered exprs +""" +function expr_to_syntaxtree(@nospecialize(e), lnn::Union{LineNumberNode, Nothing}=nothing) + graph = ensure_attributes!( + SyntaxGraph(), + kind=Kind, syntax_flags=UInt16, + source=SourceAttrType, var_id=Int, value=Any, + name_val=String, is_toplevel_thunk=Bool) + expr_to_syntaxtree(graph, e, lnn) +end + +function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing}) + graph = syntax_graph(ctx) + toplevel_src = if isnothing(lnn) + # Provenance sinkhole for all nodes until we hit a linenode + dummy_src = SourceRef( + SourceFile("No source for expression: $e"), + 1, JS.GreenNode(K"None", 0)) + _insert_tree_node(graph, K"None", dummy_src) + else + lnn + end + st_id, _ = _insert_convert_expr(e, graph, toplevel_src) + out = SyntaxTree(graph, st_id) + return out +end + +function _expr_replace!(@nospecialize(e), replace_pred::Function, replacer!::Function, + recurse_pred=(@nospecialize e)->true) + if replace_pred(e) + replacer!(e) + end + if e isa Expr && recurse_pred(e) + for a in e.args + _expr_replace!(a, replace_pred, replacer!, recurse_pred) + end + end +end + +function _to_iterspec(exs::Vector, is_generator::Bool) + if length(exs) === 1 && exs[1].head === :filter + @assert length(exs[1].args) >= 2 + return Expr(:filter, _to_iterspec(exs[1].args[2:end], true), exs[1].args[1]) + end + outex = Expr(:iteration) + for e in exs + if e.head === :block && !is_generator + for iter in e.args + push!(outex.args, Expr(:in, iter.args...)) + end + elseif e.head === :(=) + push!(outex.args, Expr(:in, e.args...)) + else + @assert false "unknown iterspec in $e" + end + end + return outex +end + +""" +Return `e.args`, but with any parameters in SyntaxTree (flattened, source) order. +Parameters are expected to be as `e.args[pos]`. + +e.g. orderings of (a,b,c;d;e;f): + Expr: (tuple (parameters (parameters (parameters f) e) d) a b c) + SyntaxTree: (tuple a b c (parameters d) (parameters e) (parameters f)) +""" +function collect_expr_parameters(e::Expr, pos::Int) + params = expr_parameters(e, pos) + isnothing(params) && return copy(e.args) + args = Any[e.args[1:pos-1]..., e.args[pos+1:end]...] + return _flatten_params!(args, params) +end +function _flatten_params!(out::Vector{Any}, p::Expr) + p1 = expr_parameters(p, 1) + if !isnothing(p1) + push!(out, Expr(:parameters, p.args[2:end]...)) + _flatten_params!(out, p1) + else + push!(out, p::Any) + end + return out +end +function expr_parameters(p::Expr, pos::Int) + if length(p.args) >= pos && + p.args[pos] isa Expr && + p.args[pos].head === :parameters + return p.args[pos] + end + return nothing +end + +""" +If `b` (usually a block) has exactly one non-LineNumberNode argument, unwrap it. +""" +function maybe_unwrap_arg(b::Expr) + e1 = findfirst(c -> !isa(c, LineNumberNode), b.args) + isnothing(e1) && return b + e2 = findfirst(c -> !isa(c, LineNumberNode), b.args[e1+1:end]) + !isnothing(e2) && return b + return b.args[e1] +end + +function maybe_extract_lnn(b, default) + !(b isa Expr) && return b + lnn_i = findfirst(a->isa(a, LineNumberNode), b.args) + return isnothing(lnn_i) ? default : b.args[lnn_i] +end + +# Get kind by string if exists. TODO relies on internals +function find_kind(s::String) + out = get(JS._kind_str_to_int, s, nothing) + return isnothing(out) ? nothing : JS.Kind(out) +end + +function is_dotted_operator(s::AbstractString) + return length(s) >= 2 && + s[1] === '.' && + JS.is_operator(something(find_kind(s[2:end]), K"None")) +end + +function is_eventually_call(e) + return e isa Expr && (e.head === :call || + e.head in (:where, :(::)) && is_eventually_call(e.args[1])) +end + +""" +Insert `e` converted to a syntaxtree into graph and recurse on children. Return +a pair (my_node_id, last_srcloc). Should not mutate `e`. + +`src` is the latest location found in the pre-order traversal, and is the line +number node to be associated with `e`. +""" +function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceAttrType) + #--------------------------------------------------------------------------- + # Non-expr types + if isnothing(e) + st_id = _insert_tree_node(graph, K"core", src; name_val="nothing") + return st_id, src + elseif e isa Symbol + st_id = _insert_tree_node(graph, K"Identifier", src; name_val=String(e)) + return st_id, src + elseif e isa QuoteNode && e.value isa Symbol + # Undo special handling from st->expr + return _insert_convert_expr(Expr(:quote, e.value), graph, src) + # elseif e isa QuoteNode + # st_id = _insert_tree_node(graph, K"inert", src) + # quote_child, _ = _insert_convert_expr(e.value, graph, src) + # setchildren!(graph, st_id, NodeId[quote_child]) + # return st_id, src + elseif e isa String + st_id = _insert_tree_node(graph, K"string", src) + id_inner = _insert_tree_node(graph, K"String", src; value=e) + setchildren!(graph, st_id, [id_inner]) + return st_id, src + elseif !(e isa Expr) + # There are other kinds we could potentially back-convert (e.g. Float), + # but Value should work fine. + st_k = e isa Integer ? K"Integer" : find_kind(string(typeof(e))) + st_id = _insert_tree_node(graph, isnothing(st_k) ? K"Value" : st_k, src; value=e) + return st_id, src + end + + #--------------------------------------------------------------------------- + # `e` is an expr. In many cases, it suffices to + # - guess that the kind name is the same as the expr head + # - add no syntax flags or attrs + # - map e.args to syntax tree children one-to-one + e::Expr + nargs = length(e.args) + maybe_kind = find_kind(string(e.head)) + st_k = isnothing(maybe_kind) ? K"None" : maybe_kind + st_flags = 0x0000 + st_attrs = Dict{Symbol, Any}() + # Note that SyntaxTree/Node differentiate 0-child non-terminals and leaves + child_exprs::Union{Nothing, Vector{Any}} = copy(e.args) + + # However, the following are (many) special cases where the kind, flags, + # children, or attributes are different from what we guessed above + if Base.isoperator(e.head) && st_k === K"None" + # e.head is an updating assignment operator (+=, .-=, etc). Non-= + # dotted ops are wrapped in a call, so we don't reach this. + s = string(e.head) + @assert s[end] === '=' && nargs === 2 + if s[1] === '.' + st_k = K".op=" + op = s[2:end-1] + else + st_k = K"op=" + op = s[1:end-1] + end + child_exprs = Any[e.args[1], Symbol(op), e.args[2]] + elseif e.head === :comparison + for i = 2:2:length(child_exprs) + op = child_exprs[i] + @assert op isa Symbol + op_s = string(op) + if is_dotted_operator(op_s) + child_exprs[i] = Expr(:., Symbol(op_s[2:end])) + end + end + elseif e.head === :macrocall + @assert nargs >= 2 + a1 = e.args[1] + child_exprs = collect_expr_parameters(e, 3) + if child_exprs[2] isa LineNumberNode + src = child_exprs[2] + end + deleteat!(child_exprs, 2) + if a1 isa Symbol + child_exprs[1] = Expr(:MacroName, a1) + elseif a1 isa Expr && a1.head === :(.) && a1.args[2] isa QuoteNode + child_exprs[1] = Expr(:(.), a1.args[1], Expr(:MacroName, a1.args[2].value)) + elseif a1 isa GlobalRef + # TODO (maybe): syntax-introduced macrocalls are listed here for + # reference. We probably don't need to convert these. + if a1.name === Symbol("@cmd") + elseif a1.name === Symbol("@doc") + elseif a1.name === Symbol("@int128_str") + elseif a1.name === Symbol("@int128_str") + elseif a1.name === Symbol("@big_str") + end + elseif a1 isa Function + # pass + else + error("Unknown macrocall form $(sprint(dump, e))") + @assert false + end + elseif e.head === Symbol("'") + @assert nargs === 1 + st_k = K"call" + child_exprs = Any[e.head, e.args[1]] + elseif e.head === :. && nargs === 2 + a2 = e.args[2] + if a2 isa Expr && a2.head === :tuple + st_k = K"dotcall" + tuple_exprs = collect_expr_parameters(a2, 1) + child_exprs = pushfirst!(tuple_exprs, e.args[1]) + elseif a2 isa QuoteNode && a2.value isa Symbol + child_exprs[2] = a2.value + elseif a2 isa Expr && a2.head === :MacroName + else + @error "Unknown 2-arg dot form" e + end + elseif e.head === :for + @assert nargs === 2 + child_exprs = Any[_to_iterspec(Any[e.args[1]], false), e.args[2]] + elseif e.head === :where + @assert nargs >= 2 + if !(e.args[2] isa Expr && e.args[2].head === :braces) + child_exprs = Any[e.args[1], Expr(:braces, e.args[2:end]...)] + end + elseif e.head in (:tuple, :vect, :braces) + child_exprs = collect_expr_parameters(e, 1) + elseif e.head in (:curly, :ref) + child_exprs = collect_expr_parameters(e, 2) + elseif e.head === :try + child_exprs = Any[e.args[1]] + # Expr: + # (try (block ...) var (block ...) [block ...] [block ...]) + # # try catch_var catch finally else + # SyntaxTree: + # (try (block ...) + # [catch var (block ...)] + # [else (block ...)] + # [finally (block ...)]) + if e.args[2] != false || e.args[3] != false + push!(child_exprs, + Expr(:catch, + e.args[2] === false ? Expr(:catch_var_placeholder) : e.args[2], + e.args[3] === false ? nothing : e.args[3])) + end + if nargs >= 5 + push!(child_exprs, Expr(:else, e.args[5])) + end + if nargs >= 4 + push!(child_exprs, + Expr(:finally, e.args[4] === false ? nothing : e.args[4])) + end + elseif e.head === :flatten || e.head === :generator + st_k = K"generator" + child_exprs = Any[] + next = e + while next.head === :flatten + @assert next.args[1].head === :generator + push!(child_exprs, _to_iterspec(next.args[1].args[2:end], true)) + next = next.args[1].args[1] + end + @assert next.head === :generator + push!(child_exprs, _to_iterspec(next.args[2:end], true)) + pushfirst!(child_exprs, next.args[1]) + elseif e.head === :ncat || e.head === :nrow + dim = popfirst!(child_exprs) + st_flags |= JS.set_numeric_flags(dim) + elseif e.head === :typed_ncat + st_flags |= JS.set_numeric_flags(e.args[2]) + deleteat!(child_exprs, 2) + elseif e.head === :(->) + @assert nargs === 2 + if e.args[1] isa Expr && e.args[1].head === :block + # Expr parsing fails to make :parameters here... + lam_args = Any[] + lam_eqs = Any[] + for a in e.args[1].args + a isa Expr && a.head === :(=) ? push!(lam_eqs, a) : push!(lam_args, a) + end + !isempty(lam_eqs) && push!(lam_args, Expr(:parameters, lam_eqs...)) + child_exprs[1] = Expr(:tuple, lam_args...) + elseif !(e.args[1] isa Expr && (e.args[1].head in (:tuple, :where))) + child_exprs[1] = Expr(:tuple, e.args[1]) + end + src = maybe_extract_lnn(e.args[2], src) + child_exprs[2] = maybe_unwrap_arg(e.args[2]) + elseif e.head === :call + child_exprs = collect_expr_parameters(e, 2) + a1 = child_exprs[1] + if a1 isa Symbol + a1s = string(a1) + if is_dotted_operator(a1s) + # non-assigning dotop like .+ or .== + st_k = K"dotcall" + child_exprs[1] = Symbol(a1s[2:end]) + end + end + elseif e.head === :function + if nargs >= 2 + src = maybe_extract_lnn(e.args[2], src) + end + elseif e.head === :(=) + if is_eventually_call(e.args[1]) + st_k = K"function" + st_flags |= JS.SHORT_FORM_FUNCTION_FLAG + src = maybe_extract_lnn(e.args[2], src) + child_exprs[2] = maybe_unwrap_arg(e.args[2]) + end + elseif e.head === :module + @assert nargs === 3 + if !e.args[1] + st_flags |= JS.BARE_MODULE_FLAG + end + child_exprs = Any[e.args[2], e.args[3]] + elseif e.head === :do + # Expr: + # (do (call f args...) (-> (tuple lam_args...) (block ...))) + # SyntaxTree: + # (call f args... (do (tuple lam_args...) (block ...))) + callargs = collect_expr_parameters(e.args[1], 2) + fname = string(callargs[1]) + if e.args[1].head === :macrocall + st_k = K"macrocall" + callargs[1] = Expr(:MacroName, callargs[1]) + else + st_k = K"call" + end + child_exprs = Any[callargs..., Expr(:do_lambda, e.args[2].args...)] + elseif e.head === :let + if nargs >= 1 && !(e.args[1] isa Expr && e.args[1].head === :block) + child_exprs[1] = Expr(:block, e.args[1]) + end + elseif e.head === :struct + e.args[1] && (st_flags |= JS.MUTABLE_FLAG) + child_exprs = child_exprs[2:end] + # TODO handle docstrings after refactor + elseif (e.head === :using || e.head === :import) + _expr_replace!(e, + (e)->(e isa Expr && e.head === :.), + (e)->(e.head = :importpath)) + elseif e.head === :kw + st_k = K"=" + elseif e.head in (:local, :global) && nargs > 1 + # Possible normalization + # child_exprs = Any[Expr(:tuple, child_exprs...)] + elseif e.head === :error + # Zero-child errors from parsing are leaf nodes. We could change this + # upstream for consistency. + if nargs === 0 + child_exprs = nothing + st_attrs[:value] = JS.ErrorVal() + st_flags |= JS.TRIVIA_FLAG + end + end + + #--------------------------------------------------------------------------- + # The following heads are not emitted from parsing, but old macros could + # produce these and they would historically be accepted by flisp lowering. + if e.head === Symbol("latestworld-if-toplevel") + st_k = K"latestworld_if_toplevel" + elseif e.head === Symbol("hygienic-scope") + st_k = K"hygienic_scope" + elseif e.head === :meta + # Messy and undocumented. Only sometimes we want a K"meta". + @assert e.args[1] isa Symbol + if e.args[1] === :nospecialize + if nargs > 2 + st_k = K"block" + # Kick the can down the road (should only be simple atoms?) + child_exprs = map(c->Expr(:meta, :nospecialize, c), child_exprs[2:end]) + else + st_id, src = _insert_convert_expr(e.args[2], graph, src) + setmeta!(SyntaxTree(graph, st_id); nospecialize=true) + return st_id, src + end + else + @assert nargs === 1 + child_exprs[1] = Expr(:quoted_symbol, e.args[1]) + end + elseif e.head === :symbolicgoto || e.head === :symboliclabel + @assert nargs === 1 + st_k = e.head === :symbolicgoto ? K"symbolic_label" : K"symbolic_goto" + st_attrs[:name_val] = string(e.args[1]) + child_exprs = nothing + elseif e.head === :inline || e.head === :noinline + @assert nargs === 1 && e.args[1] isa Bool + # TODO: JuliaLowering doesn't accept this (non-:meta) form yet + st_k = K"TOMBSTONE" + child_exprs = nothing + elseif e.head === :core + @assert nargs === 1 + @assert e.args[1] isa Symbol + st_attrs[:name_val] = string(e.args[1]) + child_exprs = nothing + elseif e.head === :islocal || e.head === :isglobal + st_k = K"extension" + child_exprs = [Expr(:quoted_symbol, e.head), e.args[1]] + elseif e.head === :block && nargs >= 1 && + e.args[1] isa Expr && e.args[1].head === :softscope + # (block (softscope true) ex) produced with every REPL prompt. + # :hardscope exists too, but should just be a let, and appears to be + # unused in the wild. + ensure_attributes!(graph; scope_type=Symbol) + st_k = K"scope_block" + st_attrs[:scope_type] = :soft + child_exprs = e.args[2:end] + end + + #--------------------------------------------------------------------------- + # Temporary heads introduced by us converting the parent expr + if e.head === :MacroName + @assert nargs === 1 + mac_name = string(e.args[1]) + mac_name = mac_name == "@__dot__" ? "@." : mac_name + st_id = _insert_tree_node(graph, K"MacroName", src, st_flags; name_val=mac_name) + return st_id, src + elseif e.head === :catch_var_placeholder + st_k = K"Placeholder" + st_attrs[:name_val] = "" + child_exprs = nothing + elseif e.head === :quoted_symbol + st_k = K"Symbol" + st_attrs[:name_val] = String(e.args[1]) + child_exprs = nothing + elseif e.head === :do_lambda + st_k = K"do" + end + + #--------------------------------------------------------------------------- + # Throw if this script isn't complete. Finally, insert a new node into the + # graph and recurse on child_exprs + if st_k === K"None" + error("Unknown expr head `$(e.head)`\n$(sprint(dump, e))") + end + + st_id = _insert_tree_node(graph, st_k, src, st_flags; st_attrs...) + + # child_exprs === nothing means we want a leaf. Note that setchildren! with + # an empty list makes a node non-leaf. + if isnothing(child_exprs) + return st_id, src + else + setflags!(graph, st_id, st_flags) + st_child_ids, last_src = _insert_child_exprs(child_exprs, graph, src) + setchildren!(graph, st_id, st_child_ids) + return st_id, last_src + end +end + +function _insert_child_exprs(child_exprs::Vector{Any}, graph::SyntaxGraph, + src::SourceAttrType) + st_child_ids = NodeId[] + last_src = src + for c in child_exprs + if c isa LineNumberNode + last_src = c + else + (c_id, c_src) = _insert_convert_expr(c, graph, last_src) + push!(st_child_ids, c_id) + last_src = something(c_src, src) + end + end + return st_child_ids, last_src +end diff --git a/src/kinds.jl b/src/kinds.jl index 45138969..efd73c24 100644 --- a/src/kinds.jl +++ b/src/kinds.jl @@ -48,6 +48,9 @@ function _register_kinds() # Internal initializer for struct types, for inner constructors/functions "new" "splatnew" + # For expr-macro compatibility; gone after expansion + "escape" + "hygienic_scope" # Catch-all for additional syntax extensions without the need to # extend `Kind`. Known extensions include: # locals, islocal diff --git a/src/syntax_graph.jl b/src/syntax_graph.jl index ff9edafc..6eb0737c 100644 --- a/src/syntax_graph.jl +++ b/src/syntax_graph.jl @@ -146,9 +146,7 @@ function sethead!(graph, id::NodeId, k::Kind) end function setflags!(graph, id::NodeId, f::UInt16) - if f != 0 - graph.syntax_flags[id] = f - end + graph.syntax_flags[id] = f end function _convert_nodes(graph::SyntaxGraph, node::SyntaxNode) @@ -246,7 +244,7 @@ function attrnames(ex::SyntaxTree) [name for (name, value) in pairs(attrs) if haskey(value, ex._id)] end -function setattr(ex::SyntaxTree; extra_attrs...) +function copy_node(ex::SyntaxTree) graph = syntax_graph(ex) id = newnode!(graph) if !is_leaf(ex) @@ -254,6 +252,11 @@ function setattr(ex::SyntaxTree; extra_attrs...) end ex2 = SyntaxTree(graph, id) copy_attrs!(ex2, ex, true) + ex2 +end + +function setattr(ex::SyntaxTree; extra_attrs...) + ex2 = copy_node(ex) setattr!(ex2; extra_attrs...) ex2 end diff --git a/test/compat.jl b/test/compat.jl new file mode 100644 index 00000000..f73628db --- /dev/null +++ b/test/compat.jl @@ -0,0 +1,390 @@ +using Test +using JuliaSyntax +using JuliaLowering +const JS = JuliaSyntax +const JL = JuliaLowering + +@testset "expr->syntaxtree" begin + @testset "semantics only" begin + # Test that `s` evaluates to the same thing both under normal parsing + # and with the expr->tree->expr transformation + + programs = [ + "let x = 2; x += 5; x -= 1; [1] .*= 1; end", + "let var\"x\" = 123; x; end", + "try; 1; catch e; e; else; 2; finally; 3; end", + "for x in 1:2, y in 3:4; x + y; end", + "[x+y for x in 1:2, y in 3:4]", + "Int[x+y for x in 1:2, y in 3:4 if true]", + "for x in 1; x+=1\n if true\n continue \n elseif false \n break\n end\n end", + "Base.Meta.@lower 1", + "function foo(x, y=1; z, what::Int=5); x + y + z + what; end; foo(1,2;z=3)", + "(()->1)()", + "((x)->2)(3)", + "((x,y)->4)(5,6)", + "filter([1,2,3]) do x; x > 1; end", + """ + struct X + f1::Int # hi + "foo" + f2::Int + f3::Int + X(y) = new(y,y,y) + end + """, + "global x,y", + "global (x,y)", + "999999999999999999999999999999999999999", + "0x00000000000000001", + "(0x00000000000000001)", + "let x = 1; 2x; end", + "let x = 1; (2)(3)x; end", + "if false\n1\nelseif true\n 3\nend", + "\"str\"", + "\"\$(\"str\")\"", + "'a'", + "'α'", + "'\\xce\\xb1'", + "let x = 1; \"\"\"\n a\n \$x\n b\n c\"\"\"; end", + "try throw(0) catch e; 1 end", + "try 0 finally 1 end", + "try throw(0) catch e; 1 finally 2 end", + "try throw(0) catch e; 1 else 2 end", + "try throw(0) catch e; 1 else 2 finally 3 end", + "try throw(0) finally 1 catch e; 2 end", + ":.+", + ":.=", + ":(.=)", + ":+=", + ":(+=)", + ":.+=", + ":(.+=)", + ] + + test_mod_1 = Module() + test_mod_2 = Module() + + for p in programs + @testset "`$p`" begin + local good_expr, good_out, test_st, test_expr, test_out + try + good_expr = JS.parseall(Expr, p; ignore_errors=true) + good_out = Core.eval(test_mod_1, good_expr) + catch e + @error "Couldn't eval the reference expression---fix your test" + rethrow(e) + end + + test_st = JuliaLowering.expr_to_syntaxtree(good_expr) + test_expr = Expr(test_st) + test_out = Core.eval(test_mod_2, test_expr) + + @test good_out == test_out + end + end + end + + # Remove any information that can't be recovered from an Expr + function normalize_st!(st) + k = JS.kind(st) + args = JS.children(st) + + if JS.is_infix_op_call(st) && (k === K"call" || k === K"dotcall") + # Infix calls are not preserved in Expr; we need to re-order the children + pre_st_args = JL.NodeId[st[2]._id, st[1]._id] + for c in st[3:end] + push!(pre_st_args, c._id) + end + pre_st_flags = (JS.flags(st) & ~JS.INFIX_FLAG) | JS.PREFIX_CALL_FLAG + JL.setchildren!(st._graph, st._id, pre_st_args) + JL.setflags!(st._graph, st._id, pre_st_flags) + elseif JS.is_postfix_op_call(st) && (k === K"call" || k === K"dotcall") + pre_st_args = JL.NodeId[st[end]._id] + for c in st[1:end-1] + push!(pre_st_args, c._id) + end + pre_st_flags = (JS.flags(st) & ~JS.POSTFIX_OP_FLAG) | JS.PREFIX_CALL_FLAG + JL.setchildren!(st._graph, st._id, pre_st_args) + JL.setflags!(st._graph, st._id, pre_st_flags) + elseif k in JS.KSet"tuple block macrocall" + JL.setflags!(st._graph, st._id, JS.flags(st) & ~JS.PARENS_FLAG) + elseif k === K"toplevel" + JL.setflags!(st._graph, st._id, JS.flags(st) & ~JS.TOPLEVEL_SEMICOLONS_FLAG) + end + + if k in JS.KSet"tuple call dotcall macrocall vect curly braces <: >:" + JL.setflags!(st._graph, st._id, JS.flags(st) & ~JS.TRAILING_COMMA_FLAG) + end + + k === K"quote" && JL.setflags!(st._graph, st._id, JS.flags(st) & ~JS.COLON_QUOTE) + k === K"wrapper" && JL.sethead!(st._graph, st._id, K"block") + + # All ops are prefix ops in an expr. + # Ignore trivia (shows up on some K"error"s) + JL.setflags!(st._graph, st._id, JS.flags(st) & + ~JS.PREFIX_OP_FLAG & ~JS.INFIX_FLAG & ~JS.TRIVIA_FLAG & ~JS.NON_TERMINAL_FLAG) + + for c in JS.children(st) + normalize_st!(c) + end + return st + end + + function st_roughly_equal(; st_good, st_test) + normalize_st!(st_good) + + if kind(st_good) === kind(st_test) === K"error" + # We could consider some sort of equivalence later, but we would + # need to specify within JS what the error node contains. + return true + end + + out = kind(st_good) === kind(st_test) && + JS.flags(st_good) === JS.flags(st_test) && + JS.numchildren(st_good) === JS.numchildren(st_test) && + JS.is_leaf(st_good) === JS.is_leaf(st_test) && + get(st_good, :value, nothing) === get(st_test, :value, nothing) && + get(st_good, :name_val, nothing) === get(st_test, :name_val, nothing) && + all(map((cg, ct)->st_roughly_equal(;st_good=cg, st_test=ct), + JS.children(st_good), JS.children(st_test))) + + !out && @warn("!st_roughly_equal (normalized_reference, st_test):", + JS.sourcetext(st_good), st_good, st_test) + return out + end + + @testset "SyntaxTree equivalence (tests taken from JuliaSyntax expr.jl)" begin + # test that string->tree->expr->tree ~= string->tree + # ^^ + programs = [ + "begin a\nb\n\nc\nend", + "(a;b;c)", + "begin end", + "(;;)", + "a;b", + "module A\n\nbody\nend", + "function f()\na\n\nb\nend", + "f() = 1", + "macro f()\na\nend", + "function f end", + "macro f end", + "function (f() where {T}) end", + "function (f()::S) end", + "a -> b", + "(a,) -> b", + "(a where {T}) -> b", + "a -> (\nb;c)", + "a -> begin\nb\nc\nend", + "(a;b=1) -> c", + "(a...;b...) -> c", + "(;) -> c", + "a::T -> b", + "let i=is, j=js\nbody\nend", + "for x=xs\n\nend", + "for x=xs\ny\nend", + "while cond\n\nend", + "while cond\ny\nend", + "f() = xs", + "f() =\n(a;b)", + "f() =\nbegin\na\nb\nend", + "let f(x) =\ng(x)=1\nend", + "f() .= xs", + "for i=is body end", + "for i=is, j=js\nbody\nend", + "f(x) do y\n body end", + "@f(x) do y body end", + "f(x; a=1) do y body end", + "g(f(x) do y\n body end)", + "f(a=1)", + "f(; b=2)", + "f(a=1; b=2)", + "f(a; b; c)", + "+(a=1,)", + "(a=1)()", + "(x=1) != 2", + "+(a=1)", + "(a=1)'", + "f.(a=1; b=2)", + "(a=1,)", + "(a=1,; b=2)", + "(a=1,; b=2; c=3)", + "x[i=j]", + "(i=j)[x]", + "x[a, b; i=j]", + "(i=j){x}", + "x{a, b; i=j}", + "[a=1,; b=2]", + "{a=1,; b=2}", + "f(a .= 1)", + "f(((a = 1)))", + "(((a = 1)),)", + "(;((a = 1)),)", + "a.b", + "a.@b x", + "f.(x,y)", + "f.(x=1)", + "f.(a=1; b=2)", + "(a=1).()", + "x .+ y", + "(x=1) .+ y", + "a .< b .< c", + "a .< (.<) .< c", + "quote .+ end", + ".+(x)", + ".+x", + "f(.+)", + "(a, .+)", + "x += y", + "x .+= y", + "x \u2212= y", + "let x=1\n end", + "let x=1 ; end", + "let x ; end", + "let x::1 ; end", + "let x=1,y=2 end", + "let x+=1 ; end", + "let ; end", + "let ; body end", + "let\na\nb\nend", + "A where {T}", + "A where {S, T}", + "A where {X, Y; Z}", + "@m\n", + "\n@m", + "@m(x; a)", + "@m(a=1; b=2)", + "@S[a,b]", + "@S[a b]", + "@S[a; b]", + "@S[a ;; b]", + "[x,y ; z]", + "[a ;;; b ;;;; c]", + "[a b ; c d]", + "[a\nb]", + "[a b]", + "[a b ; c d]", + "T[a ;;; b ;;;; c]", + "T[a b ; c d]", + "T[a\nb]", + "T[a b]", + "T[a b ; c d]", + "(x for a in as for b in bs)", + "(x for a in as, b in bs)", + "(x for a in as, b in bs if z)", + "(x for a in as, b in bs for c in cs, d in ds)", + "(x for a in as for b in bs if z)", + "(x for a in as if z for b in bs)", + "[x for a = as for b = bs if cond1 for c = cs if cond2]" , + "[x for a = as if begin cond2 end]" , + "(x for a in as if z)", + "return x", + "struct A end", + "mutable struct A end", + "struct A <: B \n a::X \n end", + "struct A \n a \n b \n end", + "struct A const a end", + "export a", + "export +, ==", + "export \n a", + "global x", + "local x", + "global x,y", + "const x,y = 1,2", + "const x = 1", + "global x ~ 1", + "global x += 1", + "(;)", + "(; a=1)", + "(; a=1; b=2)", + "(a; b; c,d)", + "module A end", + "baremodule A end", + "import A", + ] + + for p in programs + @testset "`$p`" begin + st_good = JS.parsestmt(JL.SyntaxTree, p; ignore_errors=true) + st_test = JL.expr_to_syntaxtree(Expr(st_good)) + @test st_roughly_equal(;st_good, st_test) + end + end + end + + @testset "provenance via scavenging for LineNumberNodes" begin + # Provenenance of a node should be the last seen LineNumberNode in the + # depth-first traversal of the Expr, or the initial line given if none + # have been seen yet. If none have been seen and no initial line was + # given, .source should still be defined on all nodes (of unspecified + # value, but hopefully a helpful value for the user.) + ex = Expr(:block, + LineNumberNode(123), + Expr(:block, + Expr(:block, LineNumberNode(456)), + Expr(:block)), + Expr(:block, + Expr(:block), + Expr(:block))) + + # No initial line provided + st = JuliaLowering.expr_to_syntaxtree(ex) + for i in length(st._graph.edge_ranges) + @test !isnothing(get(SyntaxTree(st._graph, i), :source, nothing)) + end + @test let lnn = st[1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][1].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][2].source; lnn isa LineNumberNode && lnn.line === 456; end + + # Same tree, but provide an initial line + st = JuliaLowering.expr_to_syntaxtree(ex, LineNumberNode(789)) + @test let lnn = st.source; lnn isa LineNumberNode && lnn.line === 789; end + @test let lnn = st[1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][1].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][2].source; lnn isa LineNumberNode && lnn.line === 456; end + + ex = parsestmt(Expr, """ + begin + try + maybe + lots + of + lines + catch exc + y + end + end""") + st = JuliaLowering.expr_to_syntaxtree(ex, LineNumberNode(1)) + + # sanity: ensure we're testing the tree we expect + @test kind(st) === K"block" + @test kind(st[1]) === K"try" + @test kind(st[1][1]) === K"block" + @test kind(st[1][1][1]) === K"Identifier" && st[1][1][1].name_val === "maybe" + @test kind(st[1][1][2]) === K"Identifier" && st[1][1][2].name_val === "lots" + @test kind(st[1][1][3]) === K"Identifier" && st[1][1][3].name_val === "of" + @test kind(st[1][1][4]) === K"Identifier" && st[1][1][4].name_val === "lines" + @test kind(st[1][2]) === K"catch" + @test kind(st[1][2][1]) === K"Identifier" && st[1][2][1].name_val === "exc" + @test kind(st[1][2][2]) === K"block" + @test kind(st[1][2][2][1]) === K"Identifier" && st[1][2][2][1].name_val === "y" + + @test let lnn = st.source; lnn isa LineNumberNode && lnn.line === 1; end + @test let lnn = st[1].source; lnn isa LineNumberNode && lnn.line === 2; end + @test let lnn = st[1][1].source; lnn isa LineNumberNode && lnn.line === 2; end + @test let lnn = st[1][1][1].source; lnn isa LineNumberNode && lnn.line === 3; end + @test let lnn = st[1][1][2].source; lnn isa LineNumberNode && lnn.line === 4; end + @test let lnn = st[1][1][3].source; lnn isa LineNumberNode && lnn.line === 5; end + @test let lnn = st[1][1][4].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][2].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][2][1].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][2][2].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][2][2][1].source; lnn isa LineNumberNode && lnn.line === 8; end + + end +end diff --git a/test/desugaring.jl b/test/desugaring.jl index 834d0fb5..66a1766b 100644 --- a/test/desugaring.jl +++ b/test/desugaring.jl @@ -11,7 +11,7 @@ test_mod = Module(:TestMod) # end # (x, y) # end -# """) ~ @ast_ [K"block" +# """) ≈ @ast_ [K"block" # [K"block" # [K"=" # "y"::K"Identifier" diff --git a/test/quoting.jl b/test/quoting.jl index 32b91730..887e7d1b 100644 --- a/test/quoting.jl +++ b/test/quoting.jl @@ -11,7 +11,7 @@ begin end end """) -@test ex ~ @ast_ [K"block" +@test ex ≈ @ast_ [K"block" [K"call" "f"::K"Identifier" 11::K"Value" @@ -107,7 +107,7 @@ let end end """) -@test ex ~ @ast_ [K"block" +@test ex ≈ @ast_ [K"block" [K"=" "x"::K"Identifier" 1::K"Integer" diff --git a/test/runtests.jl b/test/runtests.jl index d628fde0..dc24047f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,4 +25,5 @@ include("utils.jl") include("quoting.jl") include("scopes.jl") include("typedefs.jl") + include("compat.jl") end diff --git a/test/utils.jl b/test/utils.jl index 0712cfe2..bf59f62e 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -44,7 +44,7 @@ macro ast_(tree) end end -function ~(ex1, ex2) +function ≈(ex1, ex2) if kind(ex1) != kind(ex2) || is_leaf(ex1) != is_leaf(ex2) return false end @@ -52,7 +52,7 @@ function ~(ex1, ex2) if numchildren(ex1) != numchildren(ex2) return false end - return all(c1 ~ c2 for (c1,c2) in zip(children(ex1), children(ex2))) + return all(c1 ≈ c2 for (c1,c2) in zip(children(ex1), children(ex2))) else return get(ex1, :value, nothing) == get(ex2, :value, nothing) && get(ex1, :name_val, nothing) == get(ex2, :name_val, nothing) @@ -92,7 +92,7 @@ end format_as_ast_macro(ex) Format AST `ex` as a Juila source code call to the `@ast_` macro for generating -test case comparisons with the `~` function. +test case comparisons with the `≈` function. """ format_as_ast_macro(ex) = format_as_ast_macro(stdout, ex)