fix quote escaping

Author: alicebob

sql/tokenizer.go

@@ -133,18 +133,20 @@ func tokenize(s string) ([]token, error) {
 				// + and - might be binary or unary. let the lexer figure that out
 				res = append(res, stoken(int(c), string(c)))
 			case '\'':
-				bt, bl := readSingleQuoted(s[i+1:])
+				bt, bl := readQuoted('\'', s[i+1:], true)
 				if bl == -1 {
 					return res, errors.New("no terminating ' found")
 				}
 				res = append(res, stoken(tLiteral, bt))
 				i += bl
 			case '"', '`', '[':
 				close := c
+				allowEscape := true
 				if close == '[' {
 					close = ']'
+					allowEscape = false
 				}
-				bt, bl := readQuoted(close, s[i+1:])
+				bt, bl := readQuoted(close, s[i+1:], allowEscape)
 				if bl == -1 {
 					return res, fmt.Errorf("no terminating %q found", close)
 				}
@@ -225,23 +227,18 @@ loop:
 	return ntoken(n), len(s)
 }
 
-// parse a 'bareword'. Opening ' is already gone. No escape sequences.
-func readSingleQuoted(s string) (string, int) {
-	for i, r := range s {
-		switch r {
-		case '\'':
-			return s[:i], i + 1
-		default:
-		}
-	}
-	return "", -1
-}
-
-// parse a quoted string until `close`. Opening char is already gone. No escape sequences.
-func readQuoted(close rune, s string) (string, int) {
+// parse a quoted string until `close`. Opening char is already gone.
+// >  A single quote within the string can be encoded by putting two single
+// > quotes in a row - as in Pascal. C-style escapes using the backslash
+// > character are not supported because they are not standard SQL.
+func readQuoted(close rune, s string, allowEscape bool) (string, int) {
 	for i, r := range s {
 		switch r {
 		case close:
+			if allowEscape && len(s) > i+1 && rune(s[i+1]) == close {
+				ss, si := readQuoted(close, s[i+2:], allowEscape)
+				return s[:i+1] + ss, i + si + 2
+			}
 			return s[:i], i + 1
 		default:
 		}

sql/tokenizer_test.go

@@ -137,6 +137,18 @@ func TestTokens(t *testing.T) {
 			stoken(tLiteral, "lit 1"),
 		},
 	)
+	testOK(
+		`'foo''bar' '''' '\n' "fo""o"`,
+		[]token{
+			stoken(tLiteral, "foo'bar"),
+			stoken(tLiteral, "'"),
+			stoken(tLiteral, `\n`),
+			stoken(tIdentifier, `fo"o`),
+		},
+	)
+	testError(`'foo''`, errors.New("no terminating ' found"))
+	testError(`[foo]]]`, errors.New("unexpected char at pos:5: ']'"))
+
 	testOK(
 		"|| * / % + - << >> & | < <= > >= = == != <> ~",
 		[]token{