Fixes for JuliaSyntax in Core / sysimage (#41)

* Fix copying of Tokenize source (clash with refactoring in #40)
* Ensure the previous default parser is restored by `enable_in_core!()`. This could be other than Core.Compiler.fl_parse if we're using JuliaSyntax.jl to develop itself :)

Author: c42f

src/hooks.jl

@@ -82,6 +82,8 @@ end
 # `Expr(:error)`.
 Base.Meta.ParseError(e::JuliaSyntax.ParseError) = e
 
+const _default_parser = Core._parse
+
 """
 Connect the JuliaSyntax parser to the Julia runtime so that it replaces the
 flisp parser for all parsing work.
@@ -97,7 +99,7 @@ function enable_in_core!(enable=true)
         close(_debug_log)
         _debug_log = nothing
     end
-    parser = enable ? core_parser_hook : Core.Compiler.fl_parse
+    parser = enable ? core_parser_hook : _default_parser
     Base.eval(Core, :(_parse = $parser))
     nothing
 end

src/parse_stream.jl

@@ -84,8 +84,8 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
 end
 
 #-------------------------------------------------------------------------------
-# Generic interface for types `T` which have kind and flags:
-# 1. Define kind(::T) and flags(::T) directly
+# Generic interface for types `T` which have kind and flags. Either:
+# 1. Define kind(::T) and flags(::T), or
 # 2. Define head(::T) to return a type like `SyntaxKind` for which `kind` and
 #    `flags` are defined
 kind(x)  = kind(head(x))

sysimage/compile.jl

@@ -17,6 +17,7 @@ cd(@__DIR__)
 rm("JuliaSyntax", force=true, recursive=true)
 mkdir("JuliaSyntax")
 cp("../src", "JuliaSyntax/src")
+cp("../Tokenize", "JuliaSyntax/Tokenize")
 cp("../test", "JuliaSyntax/test")
 projstr = replace(read("../Project.toml", String),
     "70703baa-626e-46a2-a12c-08ffd08c73b4"=>"54354a4c-6cac-4c00-8566-e7c1beb8bfd8")