Fix string escape sequences in lexer (#209)

Author: waterdrag0n

parser/lexer.go

@@ -249,10 +249,28 @@ func (l *Lexer) consumeMultiLineComment() {
 func (l *Lexer) consumeString() error {
 	i := 1
 	endChar := byte('\'')
-	for l.peekOk(i) && l.peekN(i) != endChar {
+	for l.peekOk(i) {
+		c := l.peekN(i)
+		// backslash escape
+		if c == '\\' {
+			i++
+			if l.peekOk(i) {
+				i++
+			}
+			continue
+		}
+		// single quote
+		if c == endChar {
+			// double single quote ''
+			if l.peekOk(i+1) && l.peekN(i+1) == endChar {
+				i += 2
+				continue
+			}
+			break
+		}
 		i++
 	}
-	if !l.peekOk(i) {
+	if !l.peekOk(i) || l.peekN(i) != endChar {
 		return errors.New("invalid string")
 	}
 	l.lastToken = &Token{

parser/lexer_test.go

@@ -32,18 +32,76 @@ func TestConsumeComment(t *testing.T) {
 }
 
 func TestConsumeString(t *testing.T) {
-	strs := []string{
-		"'hello world'",
-		"'123'",
-	}
-	for _, s := range strs {
-		lexer := NewLexer(s)
-		err := lexer.consumeToken()
-		require.NoError(t, err)
-		require.Equal(t, TokenKindString, lexer.lastToken.Kind)
-		require.Equal(t, strings.Trim(s, "'"), lexer.lastToken.String)
-		require.True(t, lexer.isEOF())
-	}
+	t.Run("Simple strings", func(t *testing.T) {
+		strs := []string{
+			"'hello world'",
+			"'123'",
+		}
+		for _, s := range strs {
+			lexer := NewLexer(s)
+			err := lexer.consumeToken()
+			require.NoError(t, err)
+			require.Equal(t, TokenKindString, lexer.lastToken.Kind)
+			require.Equal(t, strings.Trim(s, "'"), lexer.lastToken.String)
+			require.True(t, lexer.isEOF())
+		}
+	})
+
+	t.Run("Strings with backslash-escaped quotes", func(t *testing.T) {
+		testCases := []struct {
+			input    string
+			expected string
+		}{
+			{`'hello\'world'`, `hello\'world`},
+			{`'test\''`, `test\'`},
+			{`'\'abc\''`, `\'abc\'`},
+		}
+		for _, tc := range testCases {
+			lexer := NewLexer(tc.input)
+			err := lexer.consumeToken()
+			require.NoError(t, err, "Failed to parse: %s", tc.input)
+			require.Equal(t, TokenKindString, lexer.lastToken.Kind)
+			require.Equal(t, tc.expected, lexer.lastToken.String)
+			require.True(t, lexer.isEOF())
+		}
+	})
+
+	t.Run("Strings with double single quotes", func(t *testing.T) {
+		testCases := []struct {
+			input    string
+			expected string
+		}{
+			{`'hello''world'`, `hello''world`},
+			{`'test''123'`, `test''123`},
+			{`'abc''def''ghi'`, `abc''def''ghi`},
+		}
+		for _, tc := range testCases {
+			lexer := NewLexer(tc.input)
+			err := lexer.consumeToken()
+			require.NoError(t, err, "Failed to parse: %s", tc.input)
+			require.Equal(t, TokenKindString, lexer.lastToken.Kind)
+			require.Equal(t, tc.expected, lexer.lastToken.String)
+			require.True(t, lexer.isEOF())
+		}
+	})
+
+	t.Run("Strings with backslash-escaped backslashes", func(t *testing.T) {
+		testCases := []struct {
+			input    string
+			expected string
+		}{
+			{`'a\\b'`, `a\\b`},
+			{`'test\\123'`, `test\\123`},
+		}
+		for _, tc := range testCases {
+			lexer := NewLexer(tc.input)
+			err := lexer.consumeToken()
+			require.NoError(t, err, "Failed to parse: %s", tc.input)
+			require.Equal(t, TokenKindString, lexer.lastToken.Kind)
+			require.Equal(t, tc.expected, lexer.lastToken.String)
+			require.True(t, lexer.isEOF())
+		}
+	})
 }
 
 func TestConsumeNumber(t *testing.T) {

parser/testdata/query/format/select_with_settings_additional_table_filters.sql

@@ -0,0 +1,22 @@
+-- Origin SQL:
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'status = 1'};
+
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'value = \'test\''};
+
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'value = ''test'''};
+
+SELECT * FROM test_table
+SETTINGS additional_table_filters={'test_table': 'id IN (\'a\', \'b\') AND status = \'active\''}
+FORMAT JSON;
+
+SELECT number, x, y FROM (SELECT number FROM system.numbers LIMIT 5) f
+ANY LEFT JOIN (SELECT x, y FROM table_1) s ON f.number = s.x
+SETTINGS additional_table_filters={'system.numbers':'number != 3', 'table_1':'x != 2'};
+
+
+-- Format SQL:
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'status = 1'};
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'value = \'test\''};
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'value = ''test'''};
+SELECT * FROM test_table SETTINGS additional_table_filters={'test_table': 'id IN (\'a\', \'b\') AND status = \'active\''} FORMAT JSON;
+SELECT number, x, y FROM (SELECT number FROM system.numbers LIMIT 5) AS f ANY LEFT JOIN (SELECT x, y FROM table_1) AS s ON f.number = s.x SETTINGS additional_table_filters={'system.numbers': 'number != 3', 'table_1': 'x != 2'};