Merge pull request #28 from JuliaLang/cjf/sysimage-fixes

Tool and fixes for compiling a JuliaSyntax sysimage

Author: c42f

README.md

@@ -103,6 +103,22 @@ julia> parseall(Expr, "(x + y)*z")
 :($(Expr(:toplevel, :((x + y) * z))))
 ```
 
+# Using JuliaSyntax as the default parser
+
+To use JuliaSyntax as the default Julia parser to `include()` files,
+parse code with `Meta.parse()`, etc, call
+
+```
+julia> JuliaSyntax.enable_in_core!()
+```
+
+This causes some startup latency, so to reduce that you can create a custom
+system image by running the code in `./sysimage/compile.jl` as a Julia script
+(or directly using the shell, on unix). Then use `julia -J $resulting_sysimage`.
+
+Using a custom sysimage has the advantage that package precompilation will also
+go through the JuliaSyntax parser.
+
 # Parser implementation
 
 Our goal is to losslessly represent the source text with a tree; this may be

src/hooks.jl

@@ -1,3 +1,5 @@
+_debug_log = nothing
+
 # Adaptor for the API/ABI expected by the Julia runtime code.
 function core_parser_hook(code, filename, lineno, offset, options)
     try
@@ -8,17 +10,29 @@ function core_parser_hook(code, filename, lineno, offset, options)
             (ptr,len) = code
             code = String(unsafe_wrap(Array, ptr, len))
         end
+        if !isnothing(_debug_log)
+            print(_debug_log, """
+                  #-#-#-------------------------------
+                  # ENTER filename=$filename, lineno=$lineno, offset=$offset, options=$options"
+                  #-#-#-------------------------------
+                  """)
+            write(_debug_log, code)
+        end
+
         io = IOBuffer(code)
         seek(io, offset)
 
         stream = ParseStream(io)
         rule = options === :all ? :toplevel : options
         if rule !== :toplevel
-            # To copy the flisp parser driver, we ignore leading trivia when
-            # parsing statements or atoms
+            # To copy the flisp parser driver, we ignore leading and trailing
+            # trivia when parsing statements or atoms
             bump_trivia(stream)
         end
         JuliaSyntax.parse(stream; rule=rule)
+        if rule !== :toplevel
+            bump_trivia(stream)
+        end
 
         if any_error(stream)
             e = Expr(:error, ParseError(SourceFile(code), stream.diagnostics))
@@ -37,6 +51,14 @@ function core_parser_hook(code, filename, lineno, offset, options)
         # of one cancel here.
         last_offset = last_byte(stream)
 
+        if !isnothing(_debug_log)
+            println(_debug_log, """
+                    #-#-#-
+                    # EXIT last_offset=$last_offset
+                    #-#-#-
+                    """)
+        end
+
         # Rewrap result in an svec for use by the C code
         return Core.svec(ex, last_offset)
     catch exc
@@ -66,17 +88,17 @@ flisp parser for all parsing work.
 
 That is, JuliaSyntax will be used for `include()` `Meta.parse()`, the REPL, etc.
 """
-function enable_in_core!()
-    # TODO: Use invoke_in_world to freeze the world age at the time this was enabled.
-    Base.eval(Core, :(_parse = $core_parser_hook))
-    nothing
-end
-
-"""
-Revert to the flisp parser for all parsing work.
-"""
-function disable_in_core!()
-    Base.eval(Core, :(_parse = Core.Compiler.fl_parse))
+function enable_in_core!(enable=true)
+    debug_filename = get(ENV, "JULIA_SYNTAX_DEBUG_FILE", nothing)
+    global _debug_log 
+    if enable && !isnothing(debug_filename)
+        _debug_log = open(debug_filename, "w")
+    elseif !enable && !isnothing(_debug_log)
+        close(_debug_log)
+        _debug_log = nothing
+    end
+    parser = enable ? core_parser_hook : Core.Compiler.fl_parse
+    Base.eval(Core, :(_parse = $parser))
     nothing
 end
 

src/value_parsing.jl

@@ -210,41 +210,39 @@ end
 # stdlib under the name `Unicode.julia_chartransform`. See
 # https://github.com/JuliaLang/julia/pull/42561
 #
-# To allow use on older Julia versions, we reproduce that logic here.
+# To allow use on older Julia versions and to workaround the bug
+# https://github.com/JuliaLang/julia/issues/45716
+# we reproduce a specialized version of that logic here.
 
 # static wrapper around user callback function
-utf8proc_custom_func(codepoint::UInt32, callback::Any) =
-    UInt32(callback(codepoint))::UInt32
+function utf8proc_custom_func(codepoint::UInt32, ::Ptr{Cvoid})::UInt32
+    (codepoint == 0x025B ? 0x03B5 :
+    codepoint == 0x00B5 ? 0x03BC :
+    codepoint == 0x00B7 ? 0x22C5 :
+    codepoint == 0x0387 ? 0x22C5 :
+    codepoint == 0x2212 ? 0x002D :
+    codepoint)
+end
 
-function utf8proc_decompose(str, options, buffer, nwords, chartransform::T) where T
-    ret = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Cvoid}, Ref{T}),
+function utf8proc_decompose(str, options, buffer, nwords)
+    ret = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Cvoid}, Ptr{Cvoid}),
                 str, sizeof(str), buffer, nwords, options,
-                @cfunction(utf8proc_custom_func, UInt32, (UInt32, Ref{T})), chartransform)
+                @cfunction(utf8proc_custom_func, UInt32, (UInt32, Ptr{Cvoid})), C_NULL)
     ret < 0 && utf8proc_error(ret)
     return ret
 end
 
-function utf8proc_map(str::Union{String,SubString{String}}, options::Integer, chartransform=identity)
-    nwords = utf8proc_decompose(str, options, C_NULL, 0, chartransform)
+function utf8proc_map(str::Union{String,SubString{String}}, options::Integer)
+    nwords = utf8proc_decompose(str, options, C_NULL, 0)
     buffer = Base.StringVector(nwords*4)
-    nwords = utf8proc_decompose(str, options, buffer, nwords, chartransform)
+    nwords = utf8proc_decompose(str, options, buffer, nwords)
     nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options)
     nbytes < 0 && utf8proc_error(nbytes)
     return String(resize!(buffer, nbytes))
 end
 
-const _julia_charmap = Dict{UInt32,UInt32}(
-    0x025B => 0x03B5,
-    0x00B5 => 0x03BC,
-    0x00B7 => 0x22C5,
-    0x0387 => 0x22C5,
-    0x2212 => 0x002D,
-)
-
-julia_chartransform(codepoint::UInt32) = get(_julia_charmap, codepoint, codepoint)
-
 function normalize_identifier(str)
     flags = Base.Unicode.UTF8PROC_STABLE | Base.Unicode.UTF8PROC_COMPOSE
-    utf8proc_map(str, flags, julia_chartransform)
+    utf8proc_map(str, flags)
 end
 

sysimage/JuliaSyntaxCore/Project.toml

@@ -0,0 +1,10 @@
+name = "JuliaSyntaxCore"
+uuid = "05e5f68f-ccd0-4d84-a81a-f557a333a331"
+authors = ["Chris Foster <[email protected]> and contributors"]
+version = "0.1.0"
+
+[compat]
+julia = "1.6"
+
+[deps]
+JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"

sysimage/JuliaSyntaxCore/src/JuliaSyntaxCore.jl

@@ -0,0 +1,26 @@
+module JuliaSyntaxCore
+
+# A tiny module to hold initialization code for JuliaSyntax.jl integration with
+# the runtime.
+
+using JuliaSyntax
+
+import Base: JLOptions
+
+function __init__()
+    # HACK! Fool the runtime into allowing us to set Core._parse, even during
+    # incremental compilation. (Ideally we'd just arrange for Core._parse to be
+    # set to the JuliaSyntax parser. But how do we signal that to the dumping
+    # code outside of the initial creation of Core?)
+    i = findfirst(==(:incremental), fieldnames(JLOptions))
+    ptr = convert(Ptr{fieldtype(JLOptions, i)},
+                  cglobal(:jl_options, JLOptions) + fieldoffset(JLOptions, i))
+    incremental = unsafe_load(ptr)
+    incremental == 0 || unsafe_store!(ptr, 0)
+
+    JuliaSyntax.enable_in_core!()
+
+    incremental == 0 || unsafe_store!(ptr, incremental)
+end
+
+end

sysimage/compile.jl

@@ -0,0 +1,34 @@
+#!/bin/bash
+#=
+[[ $1 == +* ]] && juliaup_arg=$1 && shift # release channel for juliaup
+exec julia ${juliaup_arg} --startup-file=no -e 'include(popfirst!(ARGS))' "$0" "$@"
+=#
+
+imgs_base_path = joinpath(first(DEPOT_PATH), "sysimages", "v$VERSION")
+mkpath(imgs_base_path)
+
+using Libdl
+
+cd(@__DIR__)
+
+using Pkg
+Pkg.activate(".")
+Pkg.develop("JuliaSyntax")
+Pkg.develop(path="./JuliaSyntaxCore")
+
+image_path = joinpath(imgs_base_path, "juliasyntax_sysimage."*Libdl.dlext)
+
+using PackageCompiler
+PackageCompiler.create_sysimage(
+    ["JuliaSyntaxCore"],
+    project=".",
+    sysimage_path=image_path,
+    precompile_execution_file="precompile_exec.jl",
+    incremental=true,
+)
+
+@info """## System image compiled!
+
+      Use it with `julia -J "$image_path"`
+      """
+

sysimage/precompile_exec.jl

@@ -0,0 +1,5 @@
+import JuliaSyntax
+Base.include(@__MODULE__(), joinpath(pkgdir(JuliaSyntax), "test", "test_utils.jl"))
+Base.include(@__MODULE__(), joinpath(pkgdir(JuliaSyntax), "test", "parser.jl"))
+JuliaSyntax.enable_in_core!()
+@info "Some parsing" Meta.parse("x+y+z-w .+ [a b c]")

test/hooks.jl

@@ -4,13 +4,15 @@
     @test Meta.parse("x + 1") == :(x + 1)
     @test Meta.parse("x + 1", 1) == (:(x + 1), 6)
 
-    # Test that parsing statements incrementally works
-    @test Meta.parse("x + 1\n(y)", 1) == (:(x + 1), 6)
-    @test Meta.parse("x + 1\n(y)", 6) == (:y, 10)
+    # Test that parsing statements incrementally works and stops after
+    # whitespace / comment trivia
+    @test Meta.parse("x + 1\n(y)\n", 1) == (:(x + 1), 7)
+    @test Meta.parse("x + 1\n(y)\n", 7) == (:y, 11)
+    @test Meta.parse(" x#==#", 1) == (:x, 7)
 
     # Check that Meta.parse throws the JuliaSyntax.ParseError rather than
     # Meta.ParseError when Core integration is enabled.
     @test_throws JuliaSyntax.ParseError Meta.parse("[x")
 
-    JuliaSyntax.disable_in_core!()
+    JuliaSyntax.enable_in_core!(false)
 end