AST: Use a single kind K"op=" for updating assignments (#530)

Make all updating assignment operators like `+=` be represented with a
single `K"op="` head, with the operator itself in infix position.
For example, `x += 1` is now parsed as

    [op=]
      x    :: Identifier
      +    :: Identifier
      y    :: Identifier

This greatly reduces the number of distinct forms here from a rather big
list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=`
`^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as
kind `K"Identifier"`, as it should. It also makes it possible to add
further unicode updating operators while keeping the AST stable.

The need for this was highlighted when working on JuliaLowering. When
using `K"+="` as a head, one needs to look up the appropriate operator
from the list of updating operators or use string munging on the Kind
itself. This is quite awkward especially as it needs special rules for
inferring the macro scope of the `+` identifier. In addition, having a
single head for this form means update operator semantics only need to
be dealt with in one place.

Author: c42f

docs/src/reference.md

@@ -80,6 +80,7 @@ class of tokenization errors and lets the parser deal with them.
 * We use flags rather than child nodes to represent the difference between `struct` and `mutable struct`, `module` and `baremodule` (#220)
 * Iterations are represented with the `iteration` and `in` heads rather than `=` within the header of a `for`. Thus `for i=is ; body end` parses to `(for (iteration (in i is)) (block body))`. Cartesian iteration as in `for a=as, b=bs body end` are represented with a nested `(iteration (in a as) (in b bs))` rather than a `block` containing `=` because these lists of iterators are neither semantically nor syntactically a sequence of statements, unlike other uses of `block`. Generators also use the `iteration` head - see information on that below.
 * Short form functions like `f(x) = x + 1` are represented with the `function` head rather than the `=` head. In this case the `SHORT_FORM_FUNCTION_FLAG` flag is set to allow the surface syntactic form to be easily distinguished from long form functions.
+* All kinds of updating assignment operators like `+=` are represented with a single `K"op="` head, with the operator itself in infix position. For example, `x += 1` is `(op= x + 1)`, where the plus token is of kind `K"Identifer"`. This greatly reduces the number of distinct forms here from a rather big list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=` `^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as kind `K"Identifier"`, as it should. It also makes it possible to add further unicode updating operators while keeping the AST stable.
 
 ## More detail on tree differences
 

src/expr.jl

@@ -232,6 +232,16 @@ function _internal_node_to_Expr(source, srcrange, head, childranges, childheads,
 
     if k == K"?"
         headsym = :if
+    elseif k == K"op=" && length(args) == 3
+        lhs = args[1]
+        op = args[2]
+        rhs = args[3]
+        headstr = string(args[2], '=')
+        if is_dotted(head)
+            headstr = '.'*headstr
+        end
+        headsym = Symbol(headstr)
+        args = Any[lhs, rhs]
     elseif k == K"macrocall"
         if length(args) >= 2
             a2 = args[2]

src/kinds.jl

@@ -293,23 +293,8 @@ register_kinds!(JuliaSyntax, 0, [
     "BEGIN_ASSIGNMENTS"
         "BEGIN_SYNTACTIC_ASSIGNMENTS"
         "="
-        "+="
-        "-="   # Also used for "−="
-        "*="
-        "/="
-        "//="
-        "|="
-        "^="
-        "÷="
-        "%="
-        "<<="
-        ">>="
-        ">>>="
-        "\\="
-        "&="
+        "op="  # Updating assignment operator ( $= %= &= *= += -= //= /= <<= >>= >>>= \= ^= |= ÷= ⊻= )
         ":="
-        "\$="
-        "⊻="
         "END_SYNTACTIC_ASSIGNMENTS"
         "~"
         "≔"

src/parse_stream.jl

@@ -871,8 +871,9 @@ end
 Bump the next token, splitting it into several pieces
 
 Tokens are defined by a number of `token_spec` of shape `(nbyte, kind, flags)`.
-The number of input bytes of the last spec is taken from the remaining bytes of
-the input token, with the associated `nbyte` ignored.
+If all `nbyte` are positive, the sum must equal the token length. If one
+`nbyte` is negative, that token is given `tok_len + nbyte` bytes and the sum of
+all `nbyte` must equal zero.
 
 This is a hack which helps resolves the occasional lexing ambiguity. For
 example
@@ -887,12 +888,14 @@ function bump_split(stream::ParseStream, split_spec::Vararg{Any, N}) where {N}
     tok = stream.lookahead[stream.lookahead_index]
     stream.lookahead_index += 1
     b = _next_byte(stream)
+    toklen = tok.next_byte - b
     for (i, (nbyte, k, f)) in enumerate(split_spec)
         h = SyntaxHead(k, f)
-        b = (i == length(split_spec)) ? tok.next_byte : b + nbyte
+        b += nbyte < 0 ? (toklen + nbyte) : nbyte
         orig_k = k == K"." ? K"." : kind(tok)
         push!(stream.tokens, SyntaxToken(h, orig_k, false, b))
     end
+    @assert tok.next_byte == b
     stream.peek_count = 0
     return position(stream)
 end

src/parser.jl

@@ -340,7 +340,7 @@ function bump_dotsplit(ps, flags=EMPTY_FLAGS;
         bump_trivia(ps)
         mark = position(ps)
         k = remap_kind != K"None" ? remap_kind : kind(t)
-        pos = bump_split(ps, (1, K".", TRIVIA_FLAG), (0, k, flags))
+        pos = bump_split(ps, (1, K".", TRIVIA_FLAG), (-1, k, flags))
         if emit_dot_node
             pos = emit(ps, mark, K".")
         end
@@ -626,7 +626,22 @@ function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where {
         # a += b   ==>  (+= a b)
         # a .= b   ==>  (.= a b)
         is_short_form_func = k == K"=" && !is_dotted(t) && was_eventually_call(ps)
-        bump(ps, TRIVIA_FLAG)
+        if k == K"op="
+            # x += y   ==>  (op= x + y)
+            # x .+= y  ==>  (.op= x + y)
+            bump_trivia(ps)
+            if is_dotted(t)
+                bump_split(ps, (1, K".", TRIVIA_FLAG),
+                           (-2, K"Identifier", EMPTY_FLAGS),  # op
+                           (1, K"=", TRIVIA_FLAG))
+            else
+                bump_split(ps,
+                           (-1, K"Identifier", EMPTY_FLAGS),  # op
+                           (1, K"=", TRIVIA_FLAG))
+            end
+        else
+            bump(ps, TRIVIA_FLAG)
+        end
         bump_trivia(ps)
         # Syntax Edition TODO: We'd like to call `down` here when
         # is_short_form_func is true, to prevent `f() = 1 = 2` from parsing.
@@ -1843,7 +1858,7 @@ function parse_resword(ps::ParseState)
             # let x::1 ; end    ==>  (let (block (::-i x 1)) (block))
             # let x ; end       ==>  (let (block x) (block))
             # let x=1,y=2 ; end ==>  (let (block (= x 1) (= y 2) (block)))
-            # let x+=1 ; end    ==>  (let (block (+= x 1)) (block))
+            # let x+=1 ; end    ==>  (let (block (op= x + 1)) (block))
             parse_comma_separated(ps, parse_eq_star)
         end
         emit(ps, m, K"block")
@@ -2571,7 +2586,7 @@ function parse_import_path(ps::ParseState)
         # Modules with operator symbol names
         # import .⋆  ==>  (import (importpath . ⋆))
         bump_trivia(ps)
-        bump_split(ps, (1,K".",EMPTY_FLAGS), (1,peek(ps),EMPTY_FLAGS))
+        bump_split(ps, (1,K".",EMPTY_FLAGS), (-1,peek(ps),EMPTY_FLAGS))
     else
         # import @x     ==>  (import (importpath @x))
         # import $A     ==>  (import (importpath ($ A)))
@@ -2599,7 +2614,12 @@ function parse_import_path(ps::ParseState)
                                 warning="space between dots in import path")
             end
             bump_trivia(ps)
-            bump_split(ps, (1,K".",TRIVIA_FLAG), (1,k,EMPTY_FLAGS))
+            m = position(ps)
+            bump_split(ps, (1,K".",TRIVIA_FLAG), (-1,k,EMPTY_FLAGS))
+            if is_syntactic_operator(k)
+                # import A.=  ==>  (import (importpath A (error =)))
+                emit(ps, m, K"error", error="syntactic operators not allowed in import")
+            end
         elseif k == K"..."
             # Import the .. operator
             # import A...  ==>  (import (importpath A ..))
@@ -3550,13 +3570,13 @@ function parse_atom(ps::ParseState, check_identifiers=true)
         bump_dotsplit(ps, emit_dot_node=true, remap_kind=
                       is_syntactic_operator(leading_kind) ? leading_kind : K"Identifier")
         if check_identifiers && !is_valid_identifier(leading_kind)
-            # +=   ==>  (error +=)
+            # +=   ==>  (error (op= +))
             # ?    ==>  (error ?)
-            # .+=  ==>  (error (. +=))
+            # .+=  ==>  (error (. (op= +)))
             emit(ps, mark, K"error", error="invalid identifier")
         else
             # Quoted syntactic operators allowed
-            # :+=  ==>  (quote-: +=)
+            # :+=  ==>  (quote-: (op= +))
         end
     elseif is_keyword(leading_kind)
         if leading_kind == K"var" && (t = peek_token(ps,2);

src/tokenize.jl

@@ -93,6 +93,7 @@ function _nondot_symbolic_operator_kinds()
         K"isa"
         K"in"
         K".'"
+        K"op="
     ])
 end
 
@@ -527,14 +528,14 @@ function _next_token(l::Lexer, c)
     elseif c == '-'
         return lex_minus(l);
     elseif c == '−' # \minus '−' treated as hyphen '-'
-        return emit(l, accept(l, '=') ? K"-=" : K"-")
+        return emit(l, accept(l, '=') ? K"op=" : K"-")
     elseif c == '`'
         return lex_backtick(l);
     elseif is_identifier_start_char(c)
         return lex_identifier(l, c)
     elseif isdigit(c)
         return lex_digit(l, K"Integer")
-    elseif (k = get(_unicode_ops, c, K"error")) != K"error"
+    elseif (k = get(_unicode_ops, c, K"None")) != K"None"
         return emit(l, k)
     else
         emit(l,
@@ -797,12 +798,12 @@ function lex_greater(l::Lexer)
     if accept(l, '>')
         if accept(l, '>')
             if accept(l, '=')
-                return emit(l, K">>>=")
+                return emit(l, K"op=")
             else # >>>?, ? not a =
                 return emit(l, K">>>")
             end
         elseif accept(l, '=')
-            return emit(l, K">>=")
+            return emit(l, K"op=")
         else
             return emit(l, K">>")
         end
@@ -819,7 +820,7 @@ end
 function lex_less(l::Lexer)
     if accept(l, '<')
         if accept(l, '=')
-            return emit(l, K"<<=")
+            return emit(l, K"op=")
         else # '<<?', ? not =, ' '
             return emit(l, K"<<")
         end
@@ -888,15 +889,15 @@ end
 
 function lex_percent(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"%=")
+        return emit(l, K"op=")
     else
         return emit(l, K"%")
     end
 end
 
 function lex_bar(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"|=")
+        return emit(l, K"op=")
     elseif accept(l, '>')
         return emit(l, K"|>")
     elseif accept(l, '|')
@@ -910,7 +911,7 @@ function lex_plus(l::Lexer)
     if accept(l, '+')
         return emit(l, K"++")
     elseif accept(l, '=')
-        return emit(l, K"+=")
+        return emit(l, K"op=")
     end
     return emit(l, K"+")
 end
@@ -925,7 +926,7 @@ function lex_minus(l::Lexer)
     elseif !l.dotop && accept(l, '>')
         return emit(l, K"->")
     elseif accept(l, '=')
-        return emit(l, K"-=")
+        return emit(l, K"op=")
     end
     return emit(l, K"-")
 end
@@ -934,35 +935,35 @@ function lex_star(l::Lexer)
     if accept(l, '*')
         return emit(l, K"Error**") # "**" is an invalid operator use ^
     elseif accept(l, '=')
-        return emit(l, K"*=")
+        return emit(l, K"op=")
     end
     return emit(l, K"*")
 end
 
 function lex_circumflex(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"^=")
+        return emit(l, K"op=")
     end
     return emit(l, K"^")
 end
 
 function lex_division(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"÷=")
+        return emit(l, K"op=")
     end
     return emit(l, K"÷")
 end
 
 function lex_dollar(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"$=")
+        return emit(l, K"op=")
     end
     return emit(l, K"$")
 end
 
 function lex_xor(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"⊻=")
+        return emit(l, K"op=")
     end
     return emit(l, K"⊻")
 end
@@ -1110,7 +1111,7 @@ function lex_amper(l::Lexer)
     if accept(l, '&')
         return emit(l, K"&&")
     elseif accept(l, '=')
-        return emit(l, K"&=")
+        return emit(l, K"op=")
     else
         return emit(l, K"&")
     end
@@ -1148,20 +1149,20 @@ end
 function lex_forwardslash(l::Lexer)
     if accept(l, '/')
         if accept(l, '=')
-            return emit(l, K"//=")
+            return emit(l, K"op=")
         else
             return emit(l, K"//")
         end
     elseif accept(l, '=')
-        return emit(l, K"/=")
+        return emit(l, K"op=")
     else
         return emit(l, K"/")
     end
 end
 
 function lex_backslash(l::Lexer)
     if accept(l, '=')
-        return emit(l, K"\=")
+        return emit(l, K"op=")
     end
     return emit(l, K"\\")
 end
@@ -1193,7 +1194,7 @@ function lex_dot(l::Lexer)
         elseif pc == '−'
             l.dotop = true
             readchar(l)
-            return emit(l, accept(l, '=') ? K"-=" : K"-")
+            return emit(l, accept(l, '=') ? K"op=" : K"-")
         elseif pc =='*'
             l.dotop = true
             readchar(l)
@@ -1222,7 +1223,7 @@ function lex_dot(l::Lexer)
             l.dotop = true
             readchar(l)
             if accept(l, '=')
-                return emit(l, K"&=")
+                return emit(l, K"op=")
             else
                 if accept(l, '&')
                     return emit(l, K"&&")

test/expr.jl

@@ -501,6 +501,16 @@
         @test parsestmt("./x", ignore_errors=true) == Expr(:call, Expr(:error, Expr(:., :/)), :x)
     end
 
+    @testset "syntactic update-assignment operators" begin
+        @test parsestmt("x += y") == Expr(:(+=), :x, :y)
+        @test parsestmt("x .+= y") == Expr(:(.+=), :x, :y)
+        @test parsestmt(":+=") == QuoteNode(Symbol("+="))
+        @test parsestmt(":(+=)") == QuoteNode(Symbol("+="))
+        @test parsestmt(":.+=") == QuoteNode(Symbol(".+="))
+        @test parsestmt(":(.+=)") == QuoteNode(Symbol(".+="))
+        @test parsestmt("x \u2212= y") == Expr(:(-=), :x, :y)
+    end
+
     @testset "let" begin
         @test parsestmt("let x=1\n end") ==
             Expr(:let, Expr(:(=), :x, 1),  Expr(:block, LineNumberNode(2)))