Merge pull request #27 from CiscoM31/empty-token

Fix tokenization of empty token

Author: sebastien-rosset

filter_parser.go

@@ -146,6 +146,9 @@ func FilterTokenizer() *Tokenizer {
 func unescapeTokenString(in string) string {
 	// The call to ReplaceAll() implements
 	// SQUOTE-in-string = SQUOTE SQUOTE ; two consecutive single quotes represent one within a string literal
+	if in == "''" {
+		return in
+	}
 	return strings.ReplaceAll(in, "''", "'")
 }
 

filter_parser_test.go

@@ -457,6 +457,24 @@ func TestFilterNotBooleanProperty(t *testing.T) {
 
 }
 
+func TestFilterEmptyStringToken(t *testing.T) {
+	tokenizer := FilterTokenizer()
+	input := "City eq ''"
+	expect := []*Token{
+		{Value: "City", Type: FilterTokenLiteral},
+		{Value: "eq", Type: FilterTokenLogical},
+		{Value: "''", Type: FilterTokenString},
+	}
+	output, err := tokenizer.Tokenize(input)
+	if err != nil {
+		t.Error(err)
+	}
+	result, err := CompareTokens(expect, output)
+	if !result {
+		t.Error(err)
+	}
+}
+
 // Note: according to ODATA ABNF notation, there must be a space between not and open parenthesis.
 // http://docs.oasis-open.org/odata/odata/v4.01/csprd03/abnf/odata-abnf-construction-rules.txt
 func TestFilterNotWithNoSpace(t *testing.T) {