Merge pull request #182 from JuliaLang/c42f/julia-1.0-support

Support for running JuliaSyntax on Julia 1.0

Author: c42f

.github/workflows/CI.yml

@@ -13,7 +13,14 @@ jobs:
       fail-fast: false
       matrix:
         version:
+          - '1.0'
+          - '1.1'
+          - '1.2'
+          - '1.3'
+          - '1.4'
+          - '1.5'
           - '1.6'
+          - '1.7'
           - '1'
           - 'nightly'
         os:
@@ -22,6 +29,26 @@ jobs:
           - windows-latest
         arch:
           - x64
+        exclude:
+          # Test all OS's on
+          # - 1.0
+          # - 1.6
+          # - 1
+          # - nightly
+          # but remove some configurations from the build matrix to reduce CI time.
+          # See https://github.com/marketplace/actions/setup-julia-environment
+          - {os: 'macOS-latest', version: '1.1'}
+          - {os: 'macOS-latest', version: '1.2'}
+          - {os: 'macOS-latest', version: '1.3'}
+          - {os: 'macOS-latest', version: '1.4'}
+          - {os: 'macOS-latest', version: '1.5'}
+          - {os: 'macOS-latest', version: '1.7'}
+          - {os: 'windows-latest', version: '1.1'}
+          - {os: 'windows-latest', version: '1.2'}
+          - {os: 'windows-latest', version: '1.3'}
+          - {os: 'windows-latest', version: '1.4'}
+          - {os: 'windows-latest', version: '1.5'}
+          - {os: 'windows-latest', version: '1.7'}
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1

Project.toml

@@ -4,7 +4,7 @@ authors = ["Chris Foster <[email protected]> and contributors"]
 version = "0.2.0"
 
 [compat]
-julia = "1.6"
+julia = "1.0"
 
 [deps]
 

src/expr.jl

@@ -45,9 +45,9 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
                       Symbol("@big_str")
             return Expr(:macrocall, GlobalRef(Core, macname), nothing, str)
         elseif kind(node) == K"core_@doc"
-            return GlobalRef(Core, :var"@doc")
+            return GlobalRef(Core, Symbol("@doc"))
         elseif kind(node) == K"core_@cmd"
-            return GlobalRef(Core, :var"@cmd")
+            return GlobalRef(Core, Symbol("@cmd"))
         elseif kind(node) == K"MacroName" && val === Symbol("@.")
             return Symbol("@__dot__")
         else

src/hooks.jl

@@ -237,8 +237,21 @@ end
 function _fl_parse_hook(code, filename, lineno, offset, options)
     @static if VERSION >= v"1.8.0-DEV.1370" # https://github.com/JuliaLang/julia/pull/43876
         return Core.Compiler.fl_parse(code, filename, lineno, offset, options)
-    else
+    elseif VERSION >= v"1.6"
         return Core.Compiler.fl_parse(code, filename, offset, options)
+    else
+        if options === :all
+            ex = Base.parse_input_line(String(code), filename=filename, depwarn=false)
+            if !Meta.isexpr(ex, :toplevel)
+                ex = Expr(:toplevel, ex)
+            end
+            return ex, sizeof(code)
+        elseif options === :statement || options == :atom
+            ex, pos = Meta.parse(code, offset+1, greedy=options==:statement, raise=false)
+            return ex, pos-1
+        else
+            error("Unknown parse options $options")
+        end
     end
 end
 
@@ -248,7 +261,7 @@ end
 # FIXME: Improve this in Base somehow?
 Base.Meta.ParseError(e::JuliaSyntax.ParseError) = e
 
-const _default_parser = Core._parse
+const _default_parser = VERSION < v"1.6" ? nothing : Core._parse
 
 """
     enable_in_core!([enable=true; freeze_world_age, debug_filename])
@@ -266,6 +279,9 @@ Keyword arguments:
 """
 function enable_in_core!(enable=true; freeze_world_age = true,
         debug_filename   = get(ENV, "JULIA_SYNTAX_DEBUG_FILE", nothing))
+    if VERSION < v"1.6"
+        error("Cannot use JuliaSyntax as the main Julia parser in Julia version $VERSION < 1.6")
+    end
     _parser_world_age[] = freeze_world_age ? Base.get_world_counter() : _latest_world
     if enable && !isnothing(debug_filename)
         _debug_log[] = open(debug_filename, "w")

src/source_files.jl

@@ -65,7 +65,8 @@ function source_line_range(source::SourceFile, byte_index;
 end
 
 function source_location(::Type{LineNumberNode}, source::SourceFile, byte_index)
-    LineNumberNode(source_line(source, byte_index), source.filename)
+    LineNumberNode(source_line(source, byte_index),
+                   isnothing(source.filename) ? nothing : Symbol(source.filename))
 end
 
 function Base.show(io::IO, ::MIME"text/plain", source::SourceFile)

src/utils.jl

@@ -1,3 +1,21 @@
+# Compatibility hacks for older Julia versions
+if VERSION < v"1.1"
+    isnothing(x) = x === nothing
+end
+if VERSION < v"1.4"
+    function only(x::AbstractVector)
+        if length(x) != 1
+            error("Collection must contain exactly 1 element")
+        end
+        return x[1]
+    end
+end
+if VERSION < v"1.5"
+    import Base.peek
+end
+
+#--------------------------------------------------
+#
 # Internal error, used as assertion failure for cases we expect can't happen.
 @noinline function internal_error(strs...)
     error("Internal error: ", strs...)

src/value_parsing.jl

@@ -122,7 +122,7 @@ end
 @inline function _unsafe_parse_float(::Type{Float64}, ptr, strsize)
     Libc.errno(0)
     endptr = Ref{Ptr{UInt8}}(C_NULL)
-    x = @ccall jl_strtod_c(ptr::Ptr{UInt8}, endptr::Ptr{Ptr{UInt8}})::Cdouble
+    x = ccall(:jl_strtod_c, Cdouble, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, endptr)
     @check endptr[] == ptr + strsize
     status = :ok
     if Libc.errno() == Libc.ERANGE
@@ -155,13 +155,13 @@ end
         # strtof seems buggy on windows and doesn't set ERANGE correctly on
         # overflow. See also
         # https://github.com/JuliaLang/julia/issues/46544
-        x = Float32(@ccall jl_strtod_c(ptr::Ptr{UInt8}, endptr::Ptr{Ptr{UInt8}})::Cdouble)
+        x = Float32(ccall(:jl_strtod_c, Cdouble, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, endptr))
         if isinf(x)
             status = :overflow
             # Underflow not detected, but that will only be a warning elsewhere.
         end
     else
-        x = @ccall jl_strtof_c(ptr::Ptr{UInt8}, endptr::Ptr{Ptr{UInt8}})::Cfloat
+        x = ccall(:jl_strtof_c, Cfloat, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, endptr)
     end
     @check endptr[] == ptr + strsize
     if Libc.errno() == Libc.ERANGE

test/parse_packages.jl

@@ -1,9 +1,11 @@
 # Full-scale parsing tests of JuliaSyntax itself, Julia Base, etc.
 
+juliasyntax_dir = joinpath(@__DIR__, "..")
 @testset "Parse JuliaSyntax" begin
-    pkgdir = joinpath(@__DIR__, "..")
-    test_parse_all_in_path(joinpath(pkgdir, "src"))
-    test_parse_all_in_path(joinpath(pkgdir, "test"))
+    test_parse_all_in_path(joinpath(juliasyntax_dir, "src"))
+end
+@testset "Parse JuliaSyntax tests" begin
+    test_parse_all_in_path(joinpath(juliasyntax_dir, "test"))
 end
 
 base_path = let

test/parser_api.jl

@@ -82,14 +82,13 @@
     end
 
     @testset "error/warning handling" begin
-        # ignore_warnings
-        parse_sexpr(s;kws...) = sprint(show, MIME("text/x.sexpression"), parse(SyntaxNode, s; kws...))
-        @test_throws JuliaSyntax.ParseError parse_sexpr("try finally catch ex end")
-        @test parse_sexpr("try finally catch ex end", ignore_warnings=true) ==
+        parseshow(s;kws...) = sprint(show, MIME("text/x.sexpression"), parse(SyntaxNode, s; kws...))
+        @test_throws JuliaSyntax.ParseError parseshow("try finally catch ex end")
+        @test parseshow("try finally catch ex end", ignore_warnings=true) ==
             "(try_finally_catch (block) false false false (block) ex (block))"
         # ignore_errors
-        @test_throws JuliaSyntax.ParseError parse_sexpr("[a; b, c]")
-        @test_throws JuliaSyntax.ParseError parse_sexpr("[a; b, c]", ignore_warnings=true)
-        @test parse_sexpr("[a; b, c]", ignore_errors=true) == "(vcat a b (error-t) c)"
+        @test_throws JuliaSyntax.ParseError parseshow("[a; b, c]")
+        @test_throws JuliaSyntax.ParseError parseshow("[a; b, c]", ignore_warnings=true)
+        @test parseshow("[a; b, c]", ignore_errors=true) == "(vcat a b (error-t) c)"
     end
 end

test/runtests.jl

@@ -7,11 +7,28 @@ using JuliaSyntax: GreenNode, SyntaxNode,
     flags, EMPTY_FLAGS, TRIVIA_FLAG, INFIX_FLAG,
     children, child, setchild!, SyntaxHead
 
+include("test_utils.jl")
+# Tests for the test_utils go here to allow the utils to be included on their
+# own without invoking the tests.
+@testset "Test tools" begin
+    @test exprs_roughly_equal(Expr(:global, :x, :y),
+                              Expr(:global, Expr(:tuple, :x, :y)))
+    @test exprs_roughly_equal(Expr(:local, :x, :y),
+                              Expr(:local, Expr(:tuple, :x, :y)))
+    @test exprs_roughly_equal(1.5,
+                              Expr(:call, :*, 1.5, :f))
+    @test exprs_roughly_equal(1.5,
+                              Expr(:call, :*, 1.5, :f0))
+    @test exprs_roughly_equal(Expr(:do, Expr(:macrocall, Symbol("@f"), LineNumberNode(1), Expr(:kw, :a, 1)),
+                                   Expr(:->, Expr(:tuple), Expr(:block, LineNumberNode(1)))),
+                              Expr(:do, Expr(:macrocall, Symbol("@f"), LineNumberNode(1), Expr(:(=), :a, 1)),
+                                   Expr(:->, Expr(:tuple), Expr(:block, LineNumberNode(1)))))
+end
+
 @testset "Tokenize" begin
     include("tokenize.jl")
 end
 
-include("test_utils.jl")
 include("parse_stream.jl")
 include("parser.jl")
 include("diagnostics.jl")
@@ -20,7 +37,13 @@ include("expr.jl")
 @testset "Parsing literals from strings" begin
     include("value_parsing.jl")
 end
-include("hooks.jl")
-include("parse_packages.jl")
 include("source_files.jl")
 
+if VERSION >= v"1.6"
+    # Tests restricted to 1.6+ due to
+    # * Core._parse hook doesn't exist on v1.5 and lower
+    # * Reference parser bugs which would need workarounds for package parse comparisons
+    include("hooks.jl")
+    include("parse_packages.jl")
+end
+