Remove DOTOP_FLAG from tokenizer and parser

LICENSE.md

@@ -1,17 +1,17 @@
 The JuliaSyntax.jl package is licensed under the MIT "Expat" License:
 
 > Copyright (c) 2021 Julia Computing and contributors
-> 
+>
 > Permission is hereby granted, free of charge, to any person obtaining a copy
 > of this software and associated documentation files (the "Software"), to deal
 > in the Software without restriction, including without limitation the rights
 > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 > copies of the Software, and to permit persons to whom the Software is
 > furnished to do so, subject to the following conditions:
-> 
+>
 > The above copyright notice and this permission notice shall be included in all
 > copies or substantial portions of the Software.
-> 
+>
 > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

src/core/parse_stream.jl

@@ -376,7 +376,6 @@ function _buffer_lookahead_tokens(lexer, lookahead)
         was_whitespace = is_whitespace(k)
         had_whitespace |= was_whitespace
         f = EMPTY_FLAGS
-        raw.dotop      && (f |= DOTOP_FLAG)
         raw.suffix     && (f |= SUFFIXED_FLAG)
         push!(lookahead, SyntaxToken(SyntaxHead(k, f), k,
                                      had_whitespace, raw.endbyte + 2))

src/integration/expr.jl

@@ -313,9 +313,13 @@ end
         op = args[2]
         rhs = args[3]
         headstr = string(args[2], '=')
-        if is_dotted(nodehead)
-            headstr = '.'*headstr
-        end
+        retexpr.head = Symbol(headstr)
+        retexpr.args = Any[lhs, rhs]
+    elseif k == K".op=" && length(args) == 3
+        lhs = args[1]
+        op = args[2]
+        rhs = args[3]
+        headstr = '.' * string(args[2], '=')
         retexpr.head = Symbol(headstr)
         retexpr.args = Any[lhs, rhs]
     elseif k == K"macrocall"

src/julia/julia_parse_stream.jl

@@ -1,6 +1,4 @@
 # Token flags - may be set for operator kinded tokens
-# Operator is dotted
-const DOTOP_FLAG = RawFlags(1<<1)
 # Operator has a suffix
 const SUFFIXED_FLAG = RawFlags(1<<2)
 
@@ -112,12 +110,6 @@ Return true for postfix operator calls such as the `'ᵀ` call node parsed from
 """
 is_postfix_op_call(x) = call_type_flags(x) == POSTFIX_OP_FLAG
 
-"""
-    is_dotted(x)
-
-Return true for dotted syntax tokens
-"""
-is_dotted(x) = has_flags(x, DOTOP_FLAG)
 
 """
     is_suffixed(x)
@@ -126,12 +118,6 @@ Return true for operators which have suffixes, such as `+₁`
 """
 is_suffixed(x) = has_flags(x, SUFFIXED_FLAG)
 
-"""
-    is_decorated(x)
-
-Return true for operators which are decorated with a dot or suffix.
-"""
-is_decorated(x) = is_dotted(x) || is_suffixed(x)
 
 """
     numeric_flags(x)
@@ -144,11 +130,7 @@ numeric_flags(x) = numeric_flags(flags(x))
 function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
     str = (is_error(kind(head)) ? untokenize(kind(head); unique=false) :
            untokenize(kind(head); unique=unique))::String
-    if is_dotted(head)
-        str = "."*str
-    end
     if include_flag_suff
-        # Ignore DOTOP_FLAG - it's represented above with . prefix
         is_trivia(head)  && (str = str*"-t")
         is_infix_op_call(head)   && (str = str*"-i")
         is_prefix_op_call(head)  && (str = str*"-pre")
@@ -313,3 +295,32 @@ function bump_split(stream::ParseStream, split_spec::Vararg{Any, N}) where {N}
     stream.peek_count = 0
     return position(stream)
 end
+
+function peek_dotted_op_token(ps, allow_whitespace=false)
+    # Peek the next token, but if it is a dot, peek the next one as well
+    t = peek_token(ps)
+    isdotted = kind(t) == K"."
+    if isdotted
+        t2 = peek_token(ps, 2)
+        if !is_operator(t2) || (!allow_whitespace && preceding_whitespace(t2))
+            isdotted = false
+        else
+            t = t2
+        end
+    end
+    return (isdotted, t)
+end
+
+function bump_dotted(ps, isdot, flags=EMPTY_FLAGS; emit_dot_node=false, remap_kind=K"None")
+    if isdot
+        if emit_dot_node
+            dotmark = position(ps)
+            bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG
+        else
+            bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG
+        end
+    end
+    pos = bump(ps, flags, remap_kind=remap_kind)
+    isdot && emit_dot_node && (pos = emit(ps, dotmark, K"."))
+    return pos
+end

src/julia/kinds.jl

@@ -293,7 +293,9 @@ register_kinds!(JuliaSyntax, 0, [
     "BEGIN_ASSIGNMENTS"
         "BEGIN_SYNTACTIC_ASSIGNMENTS"
         "="
+        ".="
         "op="  # Updating assignment operator ( $= %= &= *= += -= //= /= <<= >>= >>>= \= ^= |= ÷= ⊻= )
+        ".op="
         ":="
         "END_SYNTACTIC_ASSIGNMENTS"
         "~"
@@ -470,11 +472,13 @@ register_kinds!(JuliaSyntax, 0, [
     # Level 4
     "BEGIN_LAZYOR"
         "||"
+        ".||"
     "END_LAZYOR"
 
     # Level 5
     "BEGIN_LAZYAND"
         "&&"
+        ".&&"
     "END_LAZYAND"
 
     # Level 6