From 8ca9f36739826313f1f81b46f14deb701df86705 Mon Sep 17 00:00:00 2001 From: Diego Augusto Molina Date: Wed, 27 Aug 2025 00:08:07 -0300 Subject: [PATCH 1/2] allow backticks to be quoted in string literals --- parser/lexer/lexer.go | 42 +++++++++++++++++++++++++++++++++----- parser/lexer/lexer_test.go | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index f417f02b..a8cf6a5b 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -242,15 +242,47 @@ func (l *Lexer) scanString(quote rune) (n int) { } func (l *Lexer) scanRawString(quote rune) (n int) { - ch := l.next() // read character after back tick - for ch != quote { - if ch == eof { + var escapedQuotes int +loop: + for { + ch := l.next() + for ch == quote && l.peek() == quote { + // skip current and next char which are the quote escape sequence + l.next() + ch = l.next() + escapedQuotes++ + } + switch ch { + case quote: + break loop + case eof: l.error("literal not terminated") return } - ch = l.next() n++ } - l.emitValue(String, l.source.String()[l.start.byte+1:l.end.byte-1]) + str := l.source.String()[l.start.byte+1 : l.end.byte-1] + + // handle simple case where no quoted backtick was found, then no allocation + // is needed for the new string + if escapedQuotes == 0 { + l.emitValue(String, str) + return + } + + var b strings.Builder + var skipped bool + b.Grow(len(str) - escapedQuotes) + for _, r := range str { + if r == quote { + if !skipped { + skipped = true + continue + } + skipped = false + } + b.WriteRune(r) + } + l.emitValue(String, b.String()) return } diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 5171f425..df3ea420 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -68,6 +68,22 @@ func TestLex(t *testing.T) { {Kind: EOF}, }, }, + { + "`escaped backticks` `` `a``b` ```` `a``` ```b` ```a````b``` ```````` ```a````` `````b```", + []Token{ + {Kind: String, Value: "escaped backticks"}, + {Kind: String, Value: ""}, + {Kind: String, Value: "a`b"}, + {Kind: String, Value: "`"}, + {Kind: String, Value: "a`"}, + {Kind: String, Value: "`b"}, + {Kind: String, Value: "`a``b`"}, + {Kind: String, Value: "```"}, + {Kind: String, Value: "`a``"}, + {Kind: String, Value: "``b`"}, + {Kind: EOF}, + }, + }, { "a and orb().val #.", []Token{ @@ -332,6 +348,26 @@ literal not terminated (1:10) | id "hello | .........^ +id ` + "`" + `hello +literal not terminated (1:10) + | id ` + "`" + `hello + | .........^ + +id ` + "`" + `hello` + "``" + ` +literal not terminated (1:12) + | id ` + "`" + `hello` + "``" + ` + | ...........^ + +id ` + "`" + `` + "``" + `hello +literal not terminated (1:12) + | id ` + "```" + `hello + | ...........^ + +id ` + "`" + `hello` + "``" + ` world +literal not terminated (1:18) + | id ` + "`" + `hello` + "``" + ` world + | .................^ + früh ♥︎ unrecognized character: U+2665 '♥' (1:6) | früh ♥︎ From c5afe8f231a57d92b02993ca4b12e508c168a694 Mon Sep 17 00:00:00 2001 From: Diego Augusto Molina Date: Wed, 27 Aug 2025 10:35:52 -0300 Subject: [PATCH 2/2] remove redundant empty string --- parser/lexer/lexer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index df3ea420..baa5aabb 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -358,7 +358,7 @@ literal not terminated (1:12) | id ` + "`" + `hello` + "``" + ` | ...........^ -id ` + "`" + `` + "``" + `hello +id ` + "```" + `hello literal not terminated (1:12) | id ` + "```" + `hello | ...........^