fully use JuliaSyntax for SyntaxErrorReport

Author: aviatesk

Project.toml

@@ -7,6 +7,7 @@ version = "0.9.14"
 CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 JuliaInterpreter = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
+JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
 LoweredCodeUtils = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -30,6 +31,7 @@ Cthulhu = "2.14.0"
 Example = "0.5.3"
 InteractiveUtils = "1.10"
 JuliaInterpreter = "0.9.36"
+JuliaSyntax = "0.4.10"
 Libdl = "1.10"
 Logging = "1.10"
 LoweredCodeUtils = "3.0.2"

src/JET.jl

@@ -52,6 +52,8 @@ using Base.Meta: ParseError, isexpr, lower
 
 using Base.Experimental: @MethodTable, @overlay
 
+using JuliaSyntax: JuliaSyntax
+
 using CodeTracking: CodeTracking
 
 using LoweredCodeUtils: LoweredCodeUtils, add_ssa_preds!, callee_matches
@@ -334,8 +336,6 @@ include("abstractinterpret/typeinfer.jl")
 
 function print_report end
 
-const JULIA_SYNTAX_ENABLED = !(Base.get_bool_env("JULIA_USE_FLISP_PARSER", false))
-
 include("toplevel/virtualprocess.jl")
 
 # results

src/toplevel/virtualprocess.jl

@@ -22,28 +22,34 @@ function Base.getproperty(er::ToplevelErrorReport, sym::Symbol)
 end
 
 struct SyntaxErrorReport <: ToplevelErrorReport
-    err::Exception
+    err::JuliaSyntax.ParseError
     file::String
     line::Int
-    function SyntaxErrorReport(@nospecialize(err), file, line)
-        isa(err, Exception) || (err = ErrorException(err))
-        return new(err, file, line)
+    function SyntaxErrorReport(err::JuliaSyntax.ParseError)
+        lnn = JuliaSyntax.source_location(LineNumberNode, err.source,
+            JuliaSyntax.first_byte(first(err.diagnostics)))
+        return new(err, String(lnn.file::Symbol), lnn.line)
     end
 end
 # don't show stacktrace for syntax errors
 print_report(io::IO, report::SyntaxErrorReport) = showerror(io, report.err)
 
+# TODO Use JuliaLowering.jl here
+struct LoweringErrorReport <: ToplevelErrorReport
+    msg::String
+    file::String
+    line::Int
+end
+# don't show stacktrace for syntax errors
+print_report(io::IO, report::LoweringErrorReport) = showerror(io, ErrorException(lazy"syntax: $(report.msg)"))
+
 # wraps general errors from actual execution
 struct ActualErrorWrapped <: ToplevelErrorReport
     err
     st::Base.StackTraces.StackTrace
     file::String
     line::Int
     function ActualErrorWrapped(@nospecialize(err), st, file, line)
-        if isa(err, ErrorException) && startswith(err.msg, "syntax: ")
-            # forward syntax error
-            return SyntaxErrorReport(err.msg, file, line)
-        end
         return new(err, st, file, line)
     end
 end
@@ -580,16 +586,19 @@ function _virtual_process!(res::VirtualProcessResult,
     end
 
     s = String(s)::String
-    toplevelex = Base.parse_input_line(s; filename)
+    parsed = try
+        JuliaSyntax.parseall(Expr, s; filename)
+    catch err
+        err isa JuliaSyntax.ParseError || rethrow(err)
+        err
+    end
 
-    if isexpr(toplevelex, (:error, :incomplete))
+    if parsed isa JuliaSyntax.ParseError
         # if there's any syntax error, try to identify all the syntax error location
-        report_syntax_errors!(res, s, filename)
-    elseif isnothing(toplevelex)
-        # just return if there is nothing to analyze
+        push!(res.toplevel_error_reports, SyntaxErrorReport(parsed))
     else
-        @assert isexpr(toplevelex, :toplevel)
-        _virtual_process!(res, toplevelex, filename, analyzer, config, context, pkg_mod_depth)
+        @assert isexpr(parsed, :toplevel)
+        _virtual_process!(res, parsed, filename, analyzer, config, context, pkg_mod_depth)
     end
 
     with_toplevel_logger(config) do @nospecialize(io)
@@ -642,8 +651,8 @@ function _virtual_process!(res::VirtualProcessResult,
             lwr = lower(mod, x)
             # here we should capture syntax errors found during lowering
             if isexpr(lwr, :error)
-                msg = first(lwr.args)
-                push!(res.toplevel_error_reports, SyntaxErrorReport(lazy"syntax: $msg", filename, lnnref[].line))
+                msg = first(lwr.args)::String
+                push!(res.toplevel_error_reports, LoweringErrorReport(msg, filename, lnnref[].line))
                 return nothing
             end
             return lwr
@@ -1530,35 +1539,6 @@ let s = string(nameof(AbstractGlobal))
     end
 end
 
-function report_syntax_errors!(res, s, filename)
-    index = line = 1
-    while begin
-            ex, nextindex = Base.Meta._parse_string(s, filename, line, index, :statement)
-            !isnothing(ex)
-        end
-        line += count(==('\n'), s[index:nextindex-1])
-        if isexpr(ex, :error)
-            err = only(ex.args)
-            if (@static JULIA_SYNTAX_ENABLED && true) && isa(err, ParseError)
-                report = SyntaxErrorReport(err, filename, line)
-            else
-                report = SyntaxErrorReport(lazy"syntax: $err", filename, line)
-            end
-        elseif isexpr(ex, :incomplete)
-            err = only(ex.args)
-            if (@static JULIA_SYNTAX_ENABLED && true) && isa(err, ParseError)
-                report = SyntaxErrorReport(err, filename, line)
-            else
-                report = SyntaxErrorReport(lazy"syntax: $err", filename, line)
-            end
-        else
-            report = nothing
-        end
-        isnothing(report) || push!(res.toplevel_error_reports, report)
-        index = nextindex
-    end
-end
-
 # a bridge to abstract interpretation
 function analyze_toplevel!(analyzer::AbstractAnalyzer, src::CodeInfo)
     # construct toplevel `MethodInstance`

test/toplevel/test_virtualprocess.jl

@@ -501,7 +501,7 @@ end
         end
 
         @test !isempty(res.res.toplevel_error_reports)
-        @test only(res.res.toplevel_error_reports) isa SyntaxErrorReport
+        @test only(res.res.toplevel_error_reports) isa LoweringErrorReport
     end
 end
 

test/ui/test_print.jl

@@ -21,23 +21,15 @@ end
         let s = String(take!(io))
             @test occursin("1 toplevel error found", s)
             @test occursin(Regex("@ $(@__FILE__):\\d"), s)
-            @static if JET.JULIA_SYNTAX_ENABLED
-                @test occursin("invalid identifier", s) || occursin("Expected `end`", s)
-            else
-                @test occursin("syntax: unexpected \"end\"", s)
-            end
+            @test occursin("invalid identifier", s)
         end
 
         res = report_text(src, "foo")
         print_reports(io, res.res.toplevel_error_reports)
         let s = String(take!(io))
             @test occursin("1 toplevel error found", s)
             @test occursin(r"@ foo:\d", s)
-            @static if JET.JULIA_SYNTAX_ENABLED
-                @test occursin("invalid identifier", s) || occursin("Expected `end`", s)
-            else
-                @test occursin("syntax: unexpected \"end\"", s)
-            end
+            @test occursin("invalid identifier", s)
         end
     end
 end