API updates: parse() -> parsestmt() + exports + parse!(rule=:all)

`JuliaSyntax.parse()` clashes with `Base.parse()` and it's really not clear
whether `parse()` should parse a single statement or a whole file
top-level. Having a less generic name `parsestmt()` helps with this.

Using the name `parsestmt()` is explicit about the fact that this parses a
single statement (not an "expression" which is ambiguous given
every Julia construct is an expression). It's also consistent with the
naming rules of `parseall()` and `parseatom()`, which already appear in
Base as `Meta.parseall` and `Meta.parseatom`.

Change to using `rule=:all` in the `parse!()` API rather than `rule=:toplevel`
because this is most consistent with the choices made in the `Core._parser`
interface and the naming of `parseall()`.

Also add a conservative list of exports that I expect "people are likely
to use", and which seem to be required as the main part of the API.
There's more to the API than this, but shoving that all into
the user's namespace doesn't seem ideal. Especially the parts which
are less certain.

Author: c42f

README.md

@@ -50,15 +50,16 @@ First, a source-ordered AST with `SyntaxNode` (`call-i` in the dump here means
 the `call` has the infix `-i` flag):
 
 ```julia
-julia> using JuliaSyntax: JuliaSyntax, SyntaxNode, GreenNode
+julia> using JuliaSyntax
 
-julia> JuliaSyntax.parse(SyntaxNode, "(x + y)*z", filename="foo.jl")
+julia> parsestmt(SyntaxNode, "(x + y)*z", filename="foo.jl")
 line:col│ tree                                   │ file_name
    1:1  │[call-i]                                │foo.jl
-   1:2  │  [call-i]
-   1:2  │    x
-   1:4  │    +
-   1:6  │    y
+   1:1  │  [parens]
+   1:2  │    [call-i]
+   1:2  │      x
+   1:4  │      +
+   1:6  │      y
    1:8  │  *
    1:9  │  z
 ```
@@ -71,16 +72,17 @@ representation, despite being important for parsing.
 
 ```julia
 julia> text = "(x + y)*z"
-       greentree = JuliaSyntax.parse(GreenNode, text)
+       greentree = parsestmt(JuliaSyntax.GreenNode, text)
      1:9      │[call]
-     1:1      │  (
-     2:6      │  [call]
-     2:2      │    Identifier           ✔
-     3:3      │    Whitespace
-     4:4      │    +                    ✔
-     5:5      │    Whitespace
-     6:6      │    Identifier           ✔
-     7:7      │  )
+     1:7      │  [parens]
+     1:1      │    (
+     2:6      │    [call]
+     2:2      │      Identifier         ✔
+     3:3      │      Whitespace
+     4:4      │      +                  ✔
+     5:5      │      Whitespace
+     6:6      │      Identifier         ✔
+     7:7      │    )
      8:8      │  *                      ✔
      9:9      │  Identifier             ✔
 ```
@@ -91,22 +93,23 @@ supplying the source text string:
 ```julia
 julia> show(stdout, MIME"text/plain"(), greentree, text)
      1:9      │[call]
-     1:1      │  (                          "("
-     2:6      │  [call]
-     2:2      │    Identifier           ✔   "x"
-     3:3      │    Whitespace               " "
-     4:4      │    +                    ✔   "+"
-     5:5      │    Whitespace               " "
-     6:6      │    Identifier           ✔   "y"
-     7:7      │  )                          ")"
+     1:7      │  [parens]
+     1:1      │    (                        "("
+     2:6      │    [call]
+     2:2      │      Identifier         ✔   "x"
+     3:3      │      Whitespace             " "
+     4:4      │      +                  ✔   "+"
+     5:5      │      Whitespace             " "
+     6:6      │      Identifier         ✔   "y"
+     7:7      │    )                        ")"
      8:8      │  *                      ✔   "*"
      9:9      │  Identifier             ✔   "z"
 ```
 
 Julia `Expr` can also be produced:
 
 ```julia
-julia> JuliaSyntax.parse(Expr, "(x + y)*z")
+julia> JuliaSyntax.parsestmt(Expr, "(x + y)*z")
 :((x + y) * z)
 ```
 

src/JuliaSyntax.jl

@@ -1,5 +1,23 @@
 module JuliaSyntax
 
+# Conservative list of exports - only export the most common/useful things
+# here.
+
+# Parsing. See also
+#   parse!(), ParseStream
+export parsestmt, parseall, parseatom
+# Tokenization
+export tokenize, Token, untokenize
+# Source file handling. See also
+#   highlight() sourcetext() source_line() source_location()
+export SourceFile
+# Expression heads/kinds. See also
+#   flags() and related predicates.
+export @K_str, kind, head
+# Syntax tree types. See also
+#   GreenNode
+export SyntaxNode
+
 # Helper utilities
 include("utils.jl")
 
@@ -26,4 +44,5 @@ include("expr.jl")
 # Hooks to integrate the parser with Base
 include("hooks.jl")
 include("precompile.jl")
+
 end

src/hooks.jl

@@ -122,7 +122,7 @@ end
 # Debug log file for dumping parsed code
 const _debug_log = Ref{Union{Nothing,IO}}(nothing)
 
-function _core_parser_hook(code, filename, lineno, offset, options)
+function _core_parser_hook(code, filename::String, lineno::Int, offset::Int, options::Symbol)
     try
         # TODO: Check that we do all this input wrangling without copying the
         # code buffer
@@ -144,8 +144,7 @@ function _core_parser_hook(code, filename, lineno, offset, options)
         seek(io, offset)
 
         stream = ParseStream(io)
-        rule = options === :all ? :toplevel : options
-        if rule === :statement || rule === :atom
+        if options === :statement || options === :atom
             # To copy the flisp parser driver:
             # * Parsing atoms      consumes leading trivia
             # * Parsing statements consumes leading+trailing trivia
@@ -157,8 +156,8 @@ function _core_parser_hook(code, filename, lineno, offset, options)
                 return Core.svec(nothing, last_byte(stream))
             end
         end
-        parse!(stream; rule=rule)
-        if rule === :statement
+        parse!(stream; rule=options)
+        if options === :statement
             bump_trivia(stream)
         end
 
@@ -342,7 +341,7 @@ function _fl_parse_string(text::AbstractString, filename::AbstractString,
     ex, offset+1
 end
 
-# Convenience functions to mirror `JuliaSyntax.parse(Expr, ...)` in simple cases.
+# Convenience functions to mirror `JuliaSyntax.parsestmt(Expr, ...)` in simple cases.
 fl_parse(::Type{Expr}, args...; kws...) = fl_parse(args...; kws...)
 fl_parseall(::Type{Expr}, args...; kws...) = fl_parseall(args...; kws...)
 

src/parser_api.jl

@@ -28,21 +28,25 @@ Base.display_error(io::IO, err::ParseError, bt) = Base.showerror(io, err, bt)
 
 
 """
-    parse!(stream::ParseStream; rule=:toplevel)
+    parse!(stream::ParseStream; rule=:all)
 
 Parse Julia source code from a [`ParseStream`](@ref) object. Output tree data
 structures may be extracted from `stream` with the [`build_tree`](@ref) function.
 
 `rule` may be any of
-* `:toplevel` (default) — parse a whole "file" of top level statements. In this
+* `:all` (default) — parse a whole "file" of top level statements. In this
   mode, the parser expects to fully consume the input.
 * `:statement` — parse a single statement, or statements separated by semicolons.
 * `:atom` — parse a single syntax "atom": a literal, identifier, or
   parenthesized expression.
 """
-function parse!(stream::ParseStream; rule::Symbol=:toplevel)
+function parse!(stream::ParseStream; rule::Symbol=:all)
+    if rule == :toplevel
+        Base.depwarn("Use of rule == :toplevel in parse!() is deprecated. use `rule=:all` instead.", :parse!)
+        rule = :all
+    end
     ps = ParseState(stream)
-    if rule === :toplevel
+    if rule === :all
         parse_toplevel(ps)
     elseif rule === :statement
         parse_stmts(ps)
@@ -56,14 +60,14 @@ function parse!(stream::ParseStream; rule::Symbol=:toplevel)
 end
 
 """
-    parse!(TreeType, io::IO; rule=:toplevel, version=VERSION)
+    parse!(TreeType, io::IO; rule=:all, version=VERSION)
 
 Parse Julia source code from a seekable `IO` object. The output is a tuple
 `(tree, diagnostics)`. When `parse!` returns, the stream `io` is positioned
 directly after the last byte which was consumed during parsing.
 """
 function parse!(::Type{TreeType}, io::IO;
-                rule::Symbol=:toplevel, version=VERSION, kws...) where {TreeType}
+                rule::Symbol=:all, version=VERSION, kws...) where {TreeType}
     stream = ParseStream(io; version=version)
     parse!(stream; rule=rule)
     tree = build_tree(TreeType, stream; kws...)
@@ -75,7 +79,7 @@ function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version=
                 ignore_trivia=true, filename=nothing, first_line=1, ignore_errors=false,
                 ignore_warnings=ignore_errors) where {T}
     stream = ParseStream(text, index; version=version)
-    if ignore_trivia && rule != :toplevel
+    if ignore_trivia && rule != :all
         bump_trivia(stream, skip_newlines=true)
         empty!(stream)
     end
@@ -100,19 +104,22 @@ function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version=
 end
 
 _parse_docs = """
-    parse(TreeType, text, [index];
-          version=VERSION,
-          ignore_trivia=true,
-          filename=nothing,
-          ignore_errors=false,
-          ignore_warnings=ignore_errors)
-
-    # Or, with the same arguments
+    # Parse a single expression/statement
+    parsestmt(TreeType, text, [index];
+              version=VERSION,
+              ignore_trivia=true,
+              filename=nothing,
+              ignore_errors=false,
+              ignore_warnings=ignore_errors)
+
+    # Parse all statements at top level (file scope)
     parseall(...)
+
+    # Parse a single syntax atom
     parseatom(...)
 
 Parse Julia source code string `text` into a data structure of type `TreeType`.
-`parse` parses a single Julia statement, `parseall` parses top level statements
+`parsestmt` parses a single Julia statement, `parseall` parses top level statements
 at file scope and `parseatom` parses a single Julia identifier or other "syntax
 atom".
 
@@ -136,16 +143,17 @@ parsing. To avoid exceptions due to warnings, use `ignore_warnings=true`. To
 also avoid exceptions due to errors, use `ignore_errors=true`.
 """
 
-parse(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:statement, true, T, text; kws...)[1]
-parseall(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:toplevel, true, T, text; kws...)[1]
-parseatom(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:atom, true, T, text; kws...)[1]
+"$_parse_docs"
+parsestmt(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:statement, true, T, text; kws...)[1]
 
-@eval @doc $_parse_docs parse
-@eval @doc $_parse_docs parseall
-@eval @doc $_parse_docs parseatom
+"$_parse_docs"
+parseall(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:all, true, T, text; kws...)[1]
 
-parse(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:statement, false, T, text, index; kws...)
-parseall(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:toplevel, false, T, text, index; kws...)
+"$_parse_docs"
+parseatom(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:atom, true, T, text; kws...)[1]
+
+parsestmt(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:statement, false, T, text, index; kws...)
+parseall(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:all, false, T, text, index; kws...)
 parseatom(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:atom, false, T, text, index; kws...)
 
 #-------------------------------------------------------------------------------
@@ -178,7 +186,7 @@ This interface works on UTF-8 encoded string or buffer data only.
 """
 function tokenize(text)
     ps = ParseStream(text)
-    parse!(ps, rule=:toplevel)
+    parse!(ps, rule=:all)
     ts = ps.tokens
     output_tokens = Token[]
     for i = 2:length(ts)
@@ -198,3 +206,5 @@ end
 function untokenize(token::Token, text::Vector{UInt8})
     text[token.range]
 end
+
+@deprecate parse parsestmt

src/source_files.jl

@@ -36,18 +36,18 @@ function SourceFile(; filename, kwargs...)
 end
 
 # Get line number of the given byte within the code
-function source_line_index(source::SourceFile, byte_index)
+function _source_line_index(source::SourceFile, byte_index)
     lineidx = searchsortedlast(source.line_starts, byte_index)
     return (lineidx < lastindex(source.line_starts)) ? lineidx : lineidx-1
 end
 _source_line(source::SourceFile, lineidx) = lineidx + source.first_line - 1
-source_line(source::SourceFile, byte_index) = _source_line(source, source_line_index(source, byte_index))
+source_line(source::SourceFile, byte_index) = _source_line(source, _source_line_index(source, byte_index))
 
 """
 Get line number and character within the line at the given byte index.
 """
 function source_location(source::SourceFile, byte_index)
-    lineidx = source_line_index(source, byte_index)
+    lineidx = _source_line_index(source, byte_index)
     i = source.line_starts[lineidx]
     column = 1
     while i < byte_index
@@ -63,7 +63,7 @@ Get byte range of the source line at byte_index, buffered by
 """
 function source_line_range(source::SourceFile, byte_index;
                            context_lines_before=0, context_lines_after=0)
-    lineidx = source_line_index(source, byte_index)
+    lineidx = _source_line_index(source, byte_index)
     fbyte = source.line_starts[max(lineidx-context_lines_before, 1)]
     lbyte = source.line_starts[min(lineidx+1+context_lines_after, end)] - 1
     fbyte,lbyte

test/benchmark.jl

@@ -15,7 +15,7 @@ end
 
 all_base_code = concat_base()
 
-b_ParseStream = @benchmark JuliaSyntax.parse!(JuliaSyntax.ParseStream(all_base_code), rule=:toplevel)
+b_ParseStream = @benchmark JuliaSyntax.parse!(JuliaSyntax.ParseStream(all_base_code), rule=:all)
 b_GreenNode   = @benchmark JuliaSyntax.parseall(JuliaSyntax.GreenNode, all_base_code)
 b_SyntaxNode  = @benchmark JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, all_base_code)
 b_Expr        = @benchmark JuliaSyntax.parseall(Expr, all_base_code)
@@ -30,5 +30,5 @@ b_Expr        = @benchmark JuliaSyntax.parseall(Expr, all_base_code)
 # Allocs.clear()
 # stream = JuliaSyntax.ParseStream(text);
 # JuliaSyntax.peek(stream);
-# Allocs.@profile sample_rate=1 JuliaSyntax.parse(stream)
+# Allocs.@profile sample_rate=1 JuliaSyntax.parsestmt(stream)
 # PProf.Allocs.pprof()