Cosmetics

Author: rschu1ze

tests/queries/0_stateless/03403_function_tokens.reference

@@ -4,6 +4,7 @@ Default tokenizer
 ['abc','def','foo','bar','baz','code','hello','world']
 ['abc','def','foo','bar','baz','code','hello','world']
 Ngram tokenizer
+['']
 ['abc','bc ','c d',' de','def']
 ['abc','bc ','c d',' de','def']
 ['abc def']

tests/queries/0_stateless/03403_function_tokens.sql

@@ -28,6 +28,7 @@ SELECT tokens('abc+ def- foo! bar? baz= code; hello: world/', 'default');
 
 SELECT 'Ngram tokenizer';
 
+SELECT tokens('', 'ngram') AS tokenized;
 SELECT tokens('abc def', 'ngram') AS tokenized;
 SELECT tokens('abc def', 'ngram', 3) AS tokenized;
 SELECT tokens('abc def', 'ngram', 8) AS tokenized;