Merge pull request #10 from c42f/cjf/fix-raw-triple-strings-backslash

Fix raw triple string dedenting with `\` before newline

Author: c42f

Tokenize/src/lexer.jl

@@ -523,12 +523,9 @@ function lex_string_chunk(l)
             c = readchar(l)
             if c == '\\'
                 n = 1
-                while true
+                while peekchar(l) == '\\'
                     readchar(l)
                     n += 1
-                    if peekchar(l) != '\\'
-                        break
-                    end
                 end
                 if peekchar(l) == state.delim && !iseven(n)
                     readchar(l)

Tokenize/test/lexer.jl

@@ -349,22 +349,38 @@ end
 
 @testset "string escaped newline whitespace" begin
     ts = collect(tokenize("\"x\\\n \ty\""))
-    @test ts[1] ~ (T.DQUOTE , "\"")
+    @test ts[1] ~ (T.DQUOTE, "\"")
     @test ts[2] ~ (T.STRING, "x")
     @test ts[3] ~ (T.WHITESPACE, "\\\n \t")
     @test ts[4] ~ (T.STRING, "y")
-    @test ts[5] ~ (T.DQUOTE , "\"")
+    @test ts[5] ~ (T.DQUOTE, "\"")
+
+    # No newline escape for raw strings
+    ts = collect(tokenize("r\"x\\\ny\""))
+    @test ts[1] ~ (T.IDENTIFIER , "r")
+    @test ts[2] ~ (T.DQUOTE, "\"")
+    @test ts[3] ~ (T.STRING, "x\\\ny")
+    @test ts[4] ~ (T.DQUOTE , "\"")
 end
 
 @testset "triple quoted string line splitting" begin
     ts = collect(tokenize("\"\"\"\nx\r\ny\rz\n\r\"\"\""))
     @test ts[1] ~ (T.TRIPLE_DQUOTE , "\"\"\"")
-    @test ts[2] ~ (T.STRING, "\n")
-    @test ts[3] ~ (T.STRING, "x\r\n")
-    @test ts[4] ~ (T.STRING, "y\r")
-    @test ts[5] ~ (T.STRING, "z\n")
-    @test ts[6] ~ (T.STRING, "\r")
-    @test ts[7] ~ (T.TRIPLE_DQUOTE, "\"\"\"")
+    @test ts[2] ~ (T.STRING        , "\n")
+    @test ts[3] ~ (T.STRING        , "x\r\n")
+    @test ts[4] ~ (T.STRING        , "y\r")
+    @test ts[5] ~ (T.STRING        , "z\n")
+    @test ts[6] ~ (T.STRING        , "\r")
+    @test ts[7] ~ (T.TRIPLE_DQUOTE , "\"\"\"")
+
+    # Also for raw strings
+    ts = collect(tokenize("r\"\"\"\nx\ny\"\"\""))
+    @test ts[1] ~ (T.IDENTIFIER    , "r")
+    @test ts[2] ~ (T.TRIPLE_DQUOTE , "\"\"\"")
+    @test ts[3] ~ (T.STRING        , "\n")
+    @test ts[4] ~ (T.STRING        , "x\n")
+    @test ts[5] ~ (T.STRING        , "y")
+    @test ts[6] ~ (T.TRIPLE_DQUOTE , "\"\"\"")
 end
 
 @testset "interpolation" begin

src/parser.jl

@@ -1601,7 +1601,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
             # Triple quoted procesing for custom strings
             # r"""\nx""" ==> (macrocall @r_str "x")
             # r"""\n x\n y"""     ==> (macrocall @r_str (string-sr "x\n" "y"))
-
+            # r"""\n x\\n y"""    ==> (macrocall @r_str (string-sr "x\\\n" "y"))
+            #
             # Use a special token kind for string and cmd macro names so the
             # names can be expanded later as necessary.
             outk = is_string_delim(k) ? K"StringMacroName" : K"CmdMacroName"

test/parser.jl

@@ -301,8 +301,9 @@ tests = [
         "x\"\""      => """(macrocall @x_str "")"""
         "x``"        => """(macrocall @x_cmd "")"""
         # Triple quoted procesing for custom strings
-        "r\"\"\"\nx\"\"\""      => raw"""(macrocall @r_str "x")"""
-        "r\"\"\"\n x\n y\"\"\"" => raw"""(macrocall @r_str (string-sr "x\n" "y"))"""
+        "r\"\"\"\nx\"\"\""        => raw"""(macrocall @r_str "x")"""
+        "r\"\"\"\n x\n y\"\"\""   => raw"""(macrocall @r_str (string-sr "x\n" "y"))"""
+        "r\"\"\"\n x\\\n y\"\"\"" => raw"""(macrocall @r_str (string-sr "x\\\n" "y"))"""
         # Macro sufficies can include keywords and numbers
         "x\"s\"y"    => """(macrocall @x_str "s" "y")"""
         "x\"s\"end"  => """(macrocall @x_str "s" "end")"""