Separate token output from internal nodes in ParseStream (#19)

This data rearrangement gives a cleaner separation between tokens (which
keep track of bytes in the source text) vs internal tree nodes (which
keep track of which tokens they cover). As a result it reduces the size
of the intermediate data structures.

As part of rewriting build_tree to use the new data structures it's also
become much faster and building the green tree no longer dominates the
parsing time (probably due to fixing some type stability issues).

Author: c42f

src/green_tree.jl

@@ -43,17 +43,22 @@ struct GreenNode{Head}
     args::Union{Tuple{},Vector{GreenNode{Head}}}
 end
 
+function GreenNode{Head}(head::Head, span::Integer) where {Head}
+    GreenNode{Head}(head, span, ())
+end
+
 function GreenNode(head::Head, span::Integer) where {Head}
     GreenNode{Head}(head, span, ())
 end
 
-function GreenNode(head::Head, span::Integer, args::Vector{GreenNode{Head}}) where {Head}
-    GreenNode{Head}(head, span, args)
+function GreenNode(head::Head, args) where {Head}
+    children = collect(GreenNode{Head}, args)
+    span = isempty(children) ? 0 : sum(x.span for x in children)
+    GreenNode{Head}(head, span, children)
 end
 
 function GreenNode(head::Head, args::GreenNode{Head}...) where {Head}
-    span = sum(x.span for x in args)
-    GreenNode{Head}(head, span, GreenNode{Head}[args...])
+    GreenNode{Head}(head, GreenNode{Head}[args...])
 end