Add placehold kind K"None" and use K"nothing" more precisely

K"nothing" now means the literal Julia `nothing`, and K"None" is used as
a placeholder in the code instead. This also allows us to remove the
redundant NothingLiteral kind.

Author: c42f

Tokenize/src/token_kinds.jl

@@ -1,4 +1,5 @@
 @enum(Kind,
+    NONE,      # Placeholder; never emitted by lexer
     ENDMARKER, # EOF
     ERROR,
     COMMENT, # aadsdsa, #= fdsf #=
@@ -51,7 +52,7 @@
 
     begin_cstparser,
         INVISIBLE_BRACKETS,
-        NOTHING,
+        NOTHING,  # A literal `nothing`
         WS,
         SEMICOLON_WS,
         NEWLINE_WS,
@@ -835,7 +836,6 @@
     # width tokens to keep the parse tree more uniform.
     begin_parser_tokens,
         TOMBSTONE,           # Empty placeholder for kind to be filled later
-        NOTHING_LITERAL,     # A literal Julia `nothing` in the AST
 
         # Macro names are modelled as a special kind of identifier because the
         # @ may not be attached to the macro name in the source (or may not be

src/parse_stream.jl

@@ -121,7 +121,7 @@ TODO: Optimize this data structure?  It's very large at the moment.
 """
 struct TaggedRange
     head::SyntaxHead # Kind,flags
-    orig_kind::Kind  # Kind of the original token for leaf tokens, or K"Nothing"
+    orig_kind::Kind  # Kind of the original token for leaf tokens, or K"None"
     first_byte::UInt32  # First byte in the input text
     last_byte::UInt32   # Last byte in the input text
     start_mark::UInt32  # Index of first emitted range which this range covers
@@ -362,7 +362,7 @@ end
 
 # Bump the next `n` tokens
 # flags and remap_kind are applied to any non-trivia tokens
-function _bump_n(stream::ParseStream, n::Integer, flags, remap_kind=K"Nothing")
+function _bump_n(stream::ParseStream, n::Integer, flags, remap_kind=K"None")
     if n <= 0
         return
     end
@@ -375,7 +375,7 @@ function _bump_n(stream::ParseStream, n::Integer, flags, remap_kind=K"Nothing")
         is_trivia = k ∈ (K"Whitespace", K"Comment", K"NewlineWs")
         f = is_trivia ? TRIVIA_FLAG : flags
         is_dotted(tok) && (f |= DOTOP_FLAG)
-        outk = (is_trivia || remap_kind == K"Nothing") ? k : remap_kind
+        outk = (is_trivia || remap_kind == K"None") ? k : remap_kind
         range = TaggedRange(SyntaxHead(outk, f), k, first_byte(tok),
                             last_byte(tok), lastindex(stream.ranges)+1)
         push!(stream.ranges, range)
@@ -393,7 +393,7 @@ end
 Shift the current token from the input to the output, adding the given flags.
 """
 function bump(stream::ParseStream, flags=EMPTY_FLAGS; skip_newlines=false,
-              error=nothing, remap_kind::Kind=K"Nothing")
+              error=nothing, remap_kind::Kind=K"None")
     emark = position(stream)
     _bump_n(stream, _lookahead_index(stream, 1, skip_newlines), flags, remap_kind)
     if !isnothing(error)
@@ -438,7 +438,7 @@ lexing ambiguities. There's no special whitespace handling — bump any
 whitespace if necessary with bump_trivia.
 """
 function bump_glue(stream::ParseStream, kind, flags, num_tokens)
-    span = TaggedRange(SyntaxHead(kind, flags), K"Nothing",
+    span = TaggedRange(SyntaxHead(kind, flags), K"None",
                        first_byte(stream.lookahead[1]),
                        last_byte(stream.lookahead[num_tokens]),
                        lastindex(stream.ranges) + 1)
@@ -541,7 +541,7 @@ should be a previous return value of `position()`.
 """
 function emit(stream::ParseStream, mark::ParseStreamPosition, kind::Kind,
               flags::RawFlags = EMPTY_FLAGS; error=nothing)
-    range = TaggedRange(SyntaxHead(kind, flags), K"Nothing", mark.input_byte,
+    range = TaggedRange(SyntaxHead(kind, flags), K"None", mark.input_byte,
                         stream.next_byte-1, mark.output_index+1)
     if !isnothing(error)
         _emit_diagnostic(stream, first_byte(range), last_byte(range), error=error)

src/parser.jl

@@ -1779,7 +1779,7 @@ function parse_resword(ps::ParseState)
         if k == K"NewlineWs" || is_closing_token(ps, k)
             # return\nx   ==>  (return nothing)
             # return)     ==>  (return nothing)
-            bump_invisible(ps, K"NothingLiteral")
+            bump_invisible(ps, K"nothing")
         else
             # return x    ==>  (return x)
             # return x,y  ==>  (return (tuple x y))
@@ -1889,7 +1889,7 @@ function parse_const_local_global(ps)
     mark = position(ps)
     scope_mark = mark
     has_const = false
-    scope_k = K"Nothing"
+    scope_k = K"None"
     k = peek(ps)
     if k in KSet`global local`
         # global x  ==>  (global x)
@@ -1947,7 +1947,7 @@ function parse_const_local_global(ps)
         min_supported_version(v"1.8", ps, beforevar_mark,
                               "`const` struct field without assignment")
     end
-    if scope_k != K"Nothing"
+    if scope_k != K"None"
         emit(ps, scope_mark, scope_k)
     end
     if has_const

src/syntax_tree.jl

@@ -55,7 +55,7 @@ function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::In
             isempty(val_range)  ?
                 Symbol(untokenize(k)) : # synthetic invisible tokens
                 Symbol(normalize_identifier(val_str))
-        elseif k == K"NothingLiteral"
+        elseif k == K"nothing"
             nothing
         elseif k == K"error"
             ErrorVal()

src/token_kinds.jl

@@ -5,6 +5,7 @@
 
 const _str_to_kind = let Ts = TzTokens
 Dict([
+"None"           =>  Ts.NONE
 "EndMarker"      =>  Ts.ENDMARKER
 "error"          =>  Ts.ERROR
 "Comment"        =>  Ts.COMMENT
@@ -53,9 +54,8 @@ Dict([
 "var"         =>  Ts.VAR
 "END_KEYWORDS" => Ts.end_keywords
 
-# FIXME: Define precisely what Nothing means; integrate better with other tokens.
 "BEGIN_CSTPARSER"    =>  Ts.begin_cstparser
-"Nothing"            =>  Ts.NOTHING
+"nothing"            =>  Ts.NOTHING
 "NewlineWs"          =>  Ts.NEWLINE_WS
 "END_CSTPARSER"      =>  Ts.end_cstparser
 
@@ -819,7 +819,6 @@ Dict([
 "BEGIN_PARSER_TOKENS" => Ts.begin_parser_tokens
 
 "TOMBSTONE"          =>  Ts.TOMBSTONE
-"NothingLiteral"     =>  Ts.NOTHING_LITERAL
 
 # Macro names are modelled as a special kind of identifier because the
 # @ may not be attached to the macro name in the source (or may not be
@@ -891,7 +890,7 @@ for kw in split("""
         vcat ncat typed_hcat typed_vcat typed_ncat row nrow generator
         filter flatten comprehension typed_comprehension
 
-        error Nothing
+        error nothing true false None
     """)
     _kind_to_str_unique[_str_to_kind[kw]] = kw
 end

test/parser.jl

@@ -1,10 +1,10 @@
 function test_parse(production, code; v=v"1.6")
     stream = ParseStream(code, version=v)
     production(JuliaSyntax.ParseState(stream))
-    t = JuliaSyntax.build_tree(GreenNode, stream, wrap_toplevel_as_kind=K"Nothing")
+    t = JuliaSyntax.build_tree(GreenNode, stream, wrap_toplevel_as_kind=K"None")
     source = SourceFile(code)
     s = SyntaxNode(source, t)
-    if JuliaSyntax.kind(s) == K"Nothing"
+    if JuliaSyntax.kind(s) == K"None"
         join([sprint(show, MIME("text/x.sexpression"), c) for c in children(s)], ' ')
     else
         sprint(show, MIME("text/x.sexpression"), s)