Tokenize: Add doc contextural keyword

Author: c42f

Tokenize/src/lexer.jl

@@ -1142,8 +1142,6 @@ function simple_hash(str)
 end
 
 kws = [
-Tokens.ABSTRACT,
-Tokens.AS,
 Tokens.BAREMODULE,
 Tokens.BEGIN,
 Tokens.BREAK,
@@ -1165,22 +1163,26 @@ Tokens.LET,
 Tokens.LOCAL,
 Tokens.MACRO,
 Tokens.MODULE,
-Tokens.MUTABLE,
-Tokens.OUTER,
-Tokens.PRIMITIVE,
 Tokens.QUOTE,
 Tokens.RETURN,
 Tokens.STRUCT,
 Tokens.TRY,
-Tokens.TYPE,
 Tokens.USING,
-Tokens.VAR,
 Tokens.WHILE,
 Tokens.IN,
 Tokens.ISA,
 Tokens.WHERE,
 Tokens.TRUE,
 Tokens.FALSE,
+
+Tokens.ABSTRACT,
+Tokens.AS,
+Tokens.DOC,
+Tokens.MUTABLE,
+Tokens.OUTER,
+Tokens.PRIMITIVE,
+Tokens.TYPE,
+Tokens.VAR,
 ]
 
 const kw_hash = Dict(simple_hash(lowercase(string(kw))) => kw for kw in kws)

Tokenize/src/token_kinds.jl

@@ -40,8 +40,8 @@
         begin_contextural_keywords,
             ABSTRACT,
             AS,
+            DOC,
             MUTABLE,
-            NEW,
             OUTER,
             PRIMITIVE,
             TYPE,

Tokenize/test/lexer.jl

@@ -234,10 +234,7 @@ end
 end
 
 @testset "keywords" begin
-      for kw in    ["function",
-                    "abstract",
-                    "as",
-                    "baremodule",
+      for kw in    ["baremodule",
                     "begin",
                     "break",
                     "catch",
@@ -248,27 +245,31 @@ end
                     "elseif",
                     "end",
                     "export",
-                    #"false",
                     "finally",
                     "for",
                     "function",
                     "global",
-                    "let",
-                    "local",
                     "if",
                     "import",
+                    "let",
+                    "local",
                     "macro",
                     "module",
-                    "mutable",
-                    "primitive",
                     "quote",
                     "return",
                     "struct",
-                    #"true",
                     "try",
-                    "type",
                     "using",
-                    "while"]
+                    "while",
+
+                    "abstract",
+                    "as",
+                    "doc",
+                    "mutable",
+                    "outer",
+                    "primitive",
+                    "type",
+                    "var"]
 
         @test T.kind(tok(kw)) == T.KEYWORD
     end
@@ -752,8 +753,8 @@ end
     @test all(s->Base.isoperator(Symbol(s)) == Tokens.isoperator(first(collect(tokenize(s))).kind), allops)
 end
 
-const all_kws = Set(["abstract",
-    "as",
+const all_kws = Set([
+    # Keywords
     "baremodule",
     "begin",
     "break",
@@ -775,22 +776,28 @@ const all_kws = Set(["abstract",
     "local",
     "macro",
     "module",
-    "mutable",
-    "outer",
-    "primitive",
     "quote",
     "return",
     "struct",
     "try",
-    "type",
     "using",
     "while",
-    "in",
-    "isa",
+    # Contextural keywords
+    "abstract",
+    "as",
+    "doc",
+    "mutable",
+    "outer",
+    "primitive",
+    "type",
     "var",
-    "where",
+    # Literals
     "true",
     "false",
+    # Word-like operators
+    "in",
+    "isa",
+    "where",
 ])
 
 function check_kw_hashes(iter)