Tokenize: Embedded whitespace trivia in strings

* Emit escaped newlines as whitespace trivia
* Split triple strings at newlines so parser can detect indentation and
  turn it into trivia.

Author: c42f

Tokenize/src/lexer.jl

@@ -478,6 +478,19 @@ function lex_string_chunk(l)
         # Start interpolation
         readchar(l)
         return emit(l, Tokens.EX_OR)
+    elseif !state.raw && pc == '\\' && (pc2 = dpeekchar(l)[2]; 
+                                        pc2 == '\r' || pc2 == '\n')
+        # Process escaped newline as whitespace
+        readchar(l)
+        readon(l)
+        readchar(l)
+        if pc2 == '\r' && peekchar(l) == '\n'
+            readchar(l)
+        end
+        while (pc = peekchar(l); pc == ' ' || pc == '\t')
+            readchar(l)
+        end
+        return emit(l, Tokens.WHITESPACE)
     elseif pc == state.delim && string_terminates(l, state.delim, state.triplestr)
         # Terminate string
         pop!(l.string_states)
@@ -493,14 +506,55 @@ function lex_string_chunk(l)
     readon(l)
     # Read a chunk of string characters
     if state.raw
-        read_raw_string(l, state.delim, state.triplestr)
+        # Raw strings treat all characters as literals with the exception that
+        # the closing quotes can be escaped with an odd number of \ characters.
+        while true
+            pc = peekchar(l)
+            if string_terminates(l, state.delim, state.triplestr) || eof(pc)
+                break
+            elseif state.triplestr && (pc == '\n' || pc == '\r')
+                # triple quoted newline splitting
+                readchar(l)
+                if pc == '\r' && peekchar(l) == '\n'
+                    readchar(l)
+                end
+                break
+            end
+            c = readchar(l)
+            if c == '\\'
+                n = 1
+                while true
+                    readchar(l)
+                    n += 1
+                    if peekchar(l) != '\\'
+                        break
+                    end
+                end
+                if peekchar(l) == state.delim && !iseven(n)
+                    readchar(l)
+                end
+            end
+        end
     else
         while true
             pc = peekchar(l)
             if pc == '$' || eof(pc)
                 break
+            elseif state.triplestr && (pc == '\n' || pc == '\r')
+                # triple quoted newline splitting
+                readchar(l)
+                if pc == '\r' && peekchar(l) == '\n'
+                    readchar(l)
+                end
+                break
             elseif pc == state.delim && string_terminates(l, state.delim, state.triplestr)
                 break
+            elseif pc == '\\'
+                # Escaped newline
+                pc2 = dpeekchar(l)[2]
+                if pc2 == '\r' || pc2 == '\n'
+                    break
+                end
             end
             c = readchar(l)
             if c == '\\'
@@ -928,44 +982,6 @@ function string_terminates(l, delim::Char, triplestr::Bool)
     end
 end
 
-function terminate_string(l, delim::Char, triplestr::Bool)
-    # @assert string_terminates(l, delim, triplestr)
-    readchar(l)
-    if triplestr
-        readchar(l)
-        readchar(l)
-        return delim == '"' ? Tokens.TRIPLE_DQUOTE : Tokens.TRIPLE_BACKTICK
-    else
-        return delim == '"' ? Tokens.DQUOTE : Tokens.BACKTICK
-    end
-end
-
-# Read a raw string for use with custom string macros
-#
-# Raw strings treat all characters as literals with the exception that the
-# closing quotes can be escaped with an odd number of \ characters.
-function read_raw_string(l::Lexer, delim::Char, triplestr::Bool)
-    while true
-        if string_terminates(l, delim, triplestr) || eof(peekchar(l))
-            return
-        end
-        c = readchar(l)
-        if c == '\\'
-            n = 1
-            while true
-                readchar(l)
-                n += 1
-                if peekchar(l) != '\\'
-                    break
-                end
-            end
-            if peekchar(l) == delim && !iseven(n)
-                readchar(l)
-            end
-        end
-    end
-end
-
 # Parse a token starting with a forward slash.
 # A '/' has been consumed
 function lex_forwardslash(l::Lexer)

Tokenize/test/lexer.jl

@@ -202,7 +202,7 @@ end
        """))
 
     kinds = [T.COMMENT, T.NEWLINE_WS,
-             T.TRIPLE_DQUOTE, T.STRING, T.TRIPLE_DQUOTE, T.NEWLINE_WS,
+             T.TRIPLE_DQUOTE, T.STRING, T.STRING, T.TRIPLE_DQUOTE, T.NEWLINE_WS,
              T.INTEGER, T.NEWLINE_WS,
              T.ENDMARKER]
     @test T.kind.(toks) == kinds
@@ -347,6 +347,26 @@ end
     @test ts[4] ~ (T.STRING     , "x \$ \\ y")
 end
 
+@testset "string escaped newline whitespace" begin
+    ts = collect(tokenize("\"x\\\n \ty\""))
+    @test ts[1] ~ (T.DQUOTE , "\"")
+    @test ts[2] ~ (T.STRING, "x")
+    @test ts[3] ~ (T.WHITESPACE, "\\\n \t")
+    @test ts[4] ~ (T.STRING, "y")
+    @test ts[5] ~ (T.DQUOTE , "\"")
+end
+
+@testset "triple quoted string line splitting" begin
+    ts = collect(tokenize("\"\"\"\nx\r\ny\rz\n\r\"\"\""))
+    @test ts[1] ~ (T.TRIPLE_DQUOTE , "\"\"\"")
+    @test ts[2] ~ (T.STRING, "\n")
+    @test ts[3] ~ (T.STRING, "x\r\n")
+    @test ts[4] ~ (T.STRING, "y\r")
+    @test ts[5] ~ (T.STRING, "z\n")
+    @test ts[6] ~ (T.STRING, "\r")
+    @test ts[7] ~ (T.TRIPLE_DQUOTE, "\"\"\"")
+end
+
 @testset "interpolation" begin
     @testset "basic" begin
         ts = collect(tokenize("\"\$x \$y\""))