Merge branch 'main' into sp/ternary-unexpected-kw

Author: pfitzseb

README.md

@@ -435,6 +435,33 @@ julia> text = "x = \"\"\"\n    \$a\n    b\"\"\""
     21:23     │    """                      "\"\"\""
 ```
 
+### Less redundant `block`s
+
+Sometimes `Expr` needs to contain redundant block constructs in order to have a
+place to store `LineNumberNode`s, but we don't need these and avoid adding them
+in several cases:
+* The right hand side of short form function syntax
+* The conditional in `elseif`
+* The body of anonymous functions after the `->`
+
+### Distinct conditional ternary expression
+
+The syntax `a ? b : c` is the same as `if a b else c` in `Expr` so macros can't
+distinguish these cases. Instead, we use a distinct expression head `K"?"` and
+lower to `Expr(:if)` during `Expr` conversion.
+
+### String nodes always wrapped in `K"string"` or `K"cmdstring"`
+
+All strings are surrounded by a node of kind `K"string"`, even non-interpolated
+literals, so `"x"` parses as `(string "x")`. This makes string handling simpler
+and more systematic because interpolations and triple strings with embedded
+trivia don't need to be treated differently. It also gives a container in which
+to attach the delimiting quotes.
+
+The same goes for command strings which are always wrapped in `K"cmdstring"`
+regardless of whether they have multiple pieces (due to triple-quoted
+dedenting) or otherwise.
+
 ## More about syntax kinds
 
 We generally track the type of syntax nodes with a syntax "kind", stored
@@ -549,6 +576,25 @@ name of compatibility, perhaps with a warning.)
   arises from `(set! pred char-hex?)` in `parse-number` accepting hex exponent
   digits, all of which are detected as invalid except for a trailing `f` when
   processed by `isnumtok_base`.
+* `begin` and `end` are not parsed as keywords when indexing. Typed comprehensions
+  initially look the same, but can be distinguished from indexing once we handle
+  a `for` token; it is safe to treat `begin` and `end` as keywords afterwards. The
+  reference parser *only* handles this well when there's a newline before `for`:
+  ```julia
+  Any[foo(i)
+      for i in x if begin
+          true
+      end
+  ]
+  ```
+  works, while
+  ```julia
+  Any[foo(i) for i in x if begin
+          true
+      end
+  ]
+  ```
+  does not. JuliaSyntax handles both cases.
 
 ## Parsing / AST oddities and warts
 
@@ -775,7 +821,7 @@ Here's a few links to relevant Julia issues.
 #### Lowering
 
 * A partial implementation of lowering in Julia https://github.com/JuliaLang/julia/pull/32201 —
-  some of this should be ported.
+  some of this should be ported. (Last commit at https://github.com/JuliaLang/julia/tree/df61138fcf97d03dcbbba10e962571af9700db56/ )
 * The closure capture problem https://github.com/JuliaLang/julia/issues/15276 —
   would be interesting to see whether we can tackle some of the harder cases in
   a new implementation.

src/expr.jl

@@ -6,6 +6,11 @@ function is_eventually_call(ex)
                                       is_eventually_call(ex.args[1]))
 end
 
+function is_stringchunk(node)
+    k = kind(node)
+    return k == K"String" || k == K"CmdString"
+end
+
 function _to_expr(node::SyntaxNode, iteration_spec=false, need_linenodes=true)
     if !haschildren(node)
         val = node.val
@@ -24,10 +29,62 @@ function _to_expr(node::SyntaxNode, iteration_spec=false, need_linenodes=true)
             return val
         end
     end
-    headstr = untokenize(head(node), include_flag_suff=false)
-    headsym = !isnothing(headstr) ? Symbol(headstr) :
-        error("Can't untokenize head of kind $(kind(node))")
+    if kind(node) == K"?"
+        headsym = :if
+    else
+        headstr = untokenize(head(node), include_flag_suff=false)
+        headsym = !isnothing(headstr) ? Symbol(headstr) :
+            error("Can't untokenize head of kind $(kind(node))")
+    end
     node_args = children(node)
+    if headsym == :string || headsym == :cmdstring
+        # Julia string literals may be interspersed with trivia in two situations:
+        # 1. Triple quoted string indentation is trivia
+        # 2. An \ before newline removes the newline and any following indentation
+        #
+        # Such trivia is eagerly removed by the reference parser, so here we
+        # concatenate adjacent string chunks together for compatibility.
+        args = Vector{Any}()
+        i = 1
+        while i <= length(node_args)
+            if is_stringchunk(node_args[i])
+                if i < length(node_args) && is_stringchunk(node_args[i+1])
+                    buf = IOBuffer()
+                    while i <= length(node_args) && is_stringchunk(node_args[i])
+                        write(buf, node_args[i].val)
+                        i += 1
+                    end
+                    push!(args, String(take!(buf)))
+                else
+                    push!(args, node_args[i].val)
+                    i += 1
+                end
+            else
+                e = _to_expr(node_args[i])
+                if e isa String && headsym == :string
+                    # Wrap interpolated literal strings in (string) so we can
+                    # distinguish them from the surrounding text (issue #38501)
+                    # Ie, "$("str")"  vs  "str"
+                    # https://github.com/JuliaLang/julia/pull/38692
+                    e = Expr(:string, e)
+                end
+                push!(args, e)
+                i += 1
+            end
+        end
+        if length(args) == 1 && args[1] isa String
+            # If there's a single string remaining after joining, we unwrap
+            # to give a string literal.
+            #   """\n  a\n  b""" ==>  "a\nb"
+            # headsym === :cmdstring follows this branch
+            return only(args)
+        else
+            @check headsym === :string
+            return Expr(headsym, args...)
+        end
+    end
+
+    # Convert children
     insert_linenums = (headsym == :block || headsym == :toplevel) && need_linenodes
     args = Vector{Any}(undef, length(node_args)*(insert_linenums ? 2 : 1))
     if headsym == :for && length(node_args) == 2
@@ -121,38 +178,6 @@ function _to_expr(node::SyntaxNode, iteration_spec=false, need_linenodes=true)
         pushfirst!(args, numeric_flags(flags(node)))
     elseif headsym == :typed_ncat
         insert!(args, 2, numeric_flags(flags(node)))
-    elseif headsym == :string && length(args) > 1
-        # Julia string literals may be interspersed with trivia in two situations:
-        # 1. Triple quoted string indentation is trivia
-        # 2. An \ before newline removes the newline and any following indentation
-        #
-        # Such trivia is eagerly removed by the reference parser, so here we
-        # concatenate adjacent string chunks together for compatibility.
-        #
-        # TODO: Manage the non-interpolation cases with String and CmdString
-        # kinds instead?
-        args2 = Vector{Any}()
-        i = 1
-        while i <= length(args)
-            if args[i] isa String && i < length(args) && args[i+1] isa String
-                buf = IOBuffer()
-                while i <= length(args) && args[i] isa String
-                    write(buf, args[i])
-                    i += 1
-                end
-                push!(args2, String(take!(buf)))
-            else
-                push!(args2, args[i])
-                i += 1
-            end
-        end
-        args = args2
-        if length(args2) == 1 && args2[1] isa String
-            # If there's a single string remaining after joining we unwrap to
-            # give a string literal.
-            # """\n  a\n  b""" ==>  "a\nb"
-            return args2[1]
-        end
     # elseif headsym == :string && length(args) == 1 && version <= (1,5)
     #   Strip string from interpolations in 1.5 and lower to preserve
     #   "hi$("ho")" ==>  (string "hi" "ho")

src/kinds.jl

@@ -873,6 +873,7 @@ const _kind_names =
         "curly"
         "inert"          # QuoteNode; not quasiquote
         "string"         # A string interior node (possibly containing interpolations)
+        "cmdstring"      # A cmd string node (containing delimiters plus string)
         "macrocall"
         "kw"             # the = in f(a=1)
         "parameters"     # the list after ; in f(; a=1)