Merge pull request ClickHouse#80262 from ClickHouse/ahmadov/gin-tokenizer-support

Support explicit parameters for GIN index definition

Author: ahmadov

docs/en/engines/table-engines/mergetree-family/invertedindexes.md

@@ -54,28 +54,34 @@ CREATE TABLE tab
 (
     `key` UInt64,
     `str` String,
-    INDEX inv_idx(str) TYPE gin(0) GRANULARITY 1
+    INDEX inv_idx(str) TYPE gin(tokenizer = 'default|ngram|noop' [, ngram_size = N] [, max_rows_per_postings_list = M]) GRANULARITY 1
 )
 ENGINE = MergeTree
 ORDER BY key
 ```
 
-where `N` specifies the tokenizer:
+where `tokenizer` specifies the tokenizer:
 
-- `gin(0)` (or shorter: `gin()`) set the tokenizer to "tokens", i.e. split strings along spaces,
-- `gin(N)` with `N` between 2 and 8 sets the tokenizer to "ngrams(N)"
+- `default` set the tokenizer to "tokens('default')", i.e. split strings along non-alphanumeric characters.
+- `ngram` set the tokenizer to "tokens('ngram')". i.e. splits strings to equal size terms.
+- `noop` set the tokenizer to "tokens('noop')", i.e. every value itself is a term.
 
-The maximum rows per postings list can be specified as the second parameter. This parameter can be used to control postings list sizes to avoid generating huge postings list files. The following variants exist:
+The ngram size can be specified via the `ngram_size` parameter. This is an optional parameter. The following variants exist:
 
-- `gin(ngrams, max_rows_per_postings_list)`: Use given max_rows_per_postings_list (assuming it is not 0)
-- `gin(ngrams, 0)`: No limitation of maximum rows per postings list
-- `gin(ngrams)`: Use a default maximum rows which is 64K.
+- `ngram_size = N`: with `N` between 2 and 8 sets the tokenizer to "tokens('ngram', N)".
+- If not specified: Use a default ngram size which is 3.
+
+The maximum rows per postings list can be specified via an optional `max_rows_per_postings_list`. This parameter can be used to control postings list sizes to avoid generating huge postings list files. The following variants exist:
+
+- `max_rows_per_postings_list = 0`: No limitation of maximum rows per postings list.
+- `max_rows_per_postings_list = M`: with `M` should be at least 8192.
+- If not specified: Use a default maximum rows which is 64K.
 
 Being a type of skipping index, full-text indexes can be dropped or added to a column after table creation:
 
 ```sql
 ALTER TABLE tab DROP INDEX inv_idx;
-ALTER TABLE tab ADD INDEX inv_idx(s) TYPE gin(2);
+ALTER TABLE tab ADD INDEX inv_idx(s) TYPE gin(tokenizer = 'default');
 ```
 
 To use the index, no special functions or syntax are required. Typical string search predicates automatically leverage the index. As

src/Interpreters/GinFilter.cpp

@@ -16,20 +16,12 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-}
-
-GinFilterParameters::GinFilterParameters(size_t ngrams_, UInt64 max_rows_per_postings_list_)
-    : ngrams(ngrams_)
+GinFilterParameters::GinFilterParameters(String tokenizer_, UInt64 max_rows_per_postings_list_)
+    : tokenizer(std::move(tokenizer_))
     , max_rows_per_postings_list(max_rows_per_postings_list_)
 {
     if (max_rows_per_postings_list == UNLIMITED_ROWS_PER_POSTINGS_LIST)
         max_rows_per_postings_list = std::numeric_limits<UInt64>::max();
-
-    if (ngrams > 8)
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "The size of full-text index filter cannot be greater than 8");
 }
 
 GinFilter::GinFilter(const GinFilterParameters & params_)

src/Interpreters/GinFilter.h

@@ -15,9 +15,9 @@ static inline constexpr UInt64 DEFAULT_MAX_ROWS_PER_POSTINGS_LIST = 64 * 1024;
 
 struct GinFilterParameters
 {
-    GinFilterParameters(size_t ngrams_, UInt64 max_rows_per_postings_list_);
+    GinFilterParameters(String tokenizer_, UInt64 max_rows_per_postings_list_);
 
-    size_t ngrams;
+    String tokenizer;
     UInt64 max_rows_per_postings_list;
 };
 

src/Storages/IndicesDescription.cpp

@@ -28,6 +28,45 @@ namespace ErrorCodes
 namespace
 {
 using ReplaceAliasToExprVisitor = InDepthNodeVisitor<ReplaceAliasByExpressionMatcher, true>;
+
+
+Tuple parseGinIndexArgumentFromAST(const ASTPtr & arguments)
+{
+    const auto & identifier = arguments->children[0]->template as<ASTIdentifier>();
+    if (identifier == nullptr)
+        throw Exception(ErrorCodes::INCORRECT_QUERY, "Expected identifier");
+
+    const auto & literal = arguments->children[1]->template as<ASTLiteral>();
+    if (literal == nullptr)
+        throw Exception(ErrorCodes::INCORRECT_QUERY, "Expected literal");
+
+    Tuple key_value_pair{};
+    key_value_pair.emplace_back(identifier->name());
+    key_value_pair.emplace_back(literal->value);
+    return key_value_pair;
+}
+
+bool parseGinIndexArgumentsFromAST(const ASTPtr & arguments, FieldVector & parsed_arguments)
+{
+    parsed_arguments.reserve(arguments->children.size());
+
+    for (const auto & argument : arguments->children)
+    {
+        if (const auto * ast_function = argument->template as<ASTFunction>();
+            ast_function && ast_function->name == "equals" && ast_function->arguments->children.size() == 2)
+        {
+            parsed_arguments.emplace_back(parseGinIndexArgumentFromAST(ast_function->arguments));
+        }
+        else
+        {
+            if (!parsed_arguments.empty())
+                throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot mix key-value pair and single argument as GIN index arguments");
+            return false;
+        }
+    }
+
+    return true;
+}
 }
 
 IndexDescription::IndexDescription(const IndexDescription & other)
@@ -129,6 +168,10 @@ IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast
 
     if (index_type && index_type->arguments)
     {
+        bool is_gin_index = index_type->name == "gin" || index_type->name == "inverted" || index_type->name == "full_text";
+        if (is_gin_index && parseGinIndexArgumentsFromAST(index_type->arguments, result.arguments))
+            return result;
+
         for (size_t i = 0; i < index_type->arguments->children.size(); ++i)
         {
             const auto & child = index_type->arguments->children[i];

src/Storages/MergeTree/MergeTreeIndexGin.cpp

@@ -16,10 +16,13 @@
 #include <Interpreters/misc.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/RPNBuilder.h>
+#include <boost/algorithm/string/predicate.hpp>
 #include <Common/OptimizedRegularExpression.h>
+#include <Core/Field.h>
+#include <Interpreters/ITokenExtractor.h>
+#include <base/types.h>
 #include <algorithm>
 
-
 namespace DB
 {
 namespace ErrorCodes
@@ -30,6 +33,10 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
+static const String ARGUMENT_TOKENIZER = "tokenizer";
+static const String ARGUMENT_NGRAM_SIZE = "ngram_size";
+static const String ARGUMENT_MAX_ROWS = "max_rows_per_postings_list";
+
 MergeTreeIndexGranuleGin::MergeTreeIndexGranuleGin(
     const String & index_name_,
     size_t columns_number,
@@ -758,44 +765,107 @@ MergeTreeIndexConditionPtr MergeTreeIndexGin::createIndexCondition(const Actions
     return std::make_shared<MergeTreeIndexConditionGin>(predicate, context, index.sample_block, gin_filter_params, token_extractor.get());
 }
 
-MergeTreeIndexPtr ginIndexCreator(const IndexDescription & index)
+namespace
 {
-    size_t ngram_length = index.arguments.empty() ? 0 : index.arguments[0].safeGet<size_t>();
-    UInt64 max_rows = index.arguments.size() < 2 ? DEFAULT_MAX_ROWS_PER_POSTINGS_LIST : index.arguments[1].safeGet<UInt64>();
-    GinFilterParameters gin_filter_params(ngram_length, max_rows);
 
-    if (ngram_length == 0)
+std::unordered_map<String, Field> convertArgumentsToOptionsMap(const FieldVector & arguments)
+{
+    std::unordered_map<String, Field> options;
+    for (const Field & argument : arguments)
     {
-        auto tokenizer = std::make_unique<SplitTokenExtractor>();
-        return std::make_shared<MergeTreeIndexGin>(index, gin_filter_params, std::move(tokenizer));
+        if (argument.getType() != Field::Types::Tuple)
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Arguments of GIN index must be key-value pair (identifier = literal)");
+        Tuple tuple = argument.template safeGet<Tuple>();
+        String key = tuple[0].safeGet<String>();
+        if (options.contains(key))
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "GIN index '{}' argument is specified more than once", key);
+        options[key] = tuple[1];
     }
-    else
+    return options;
+}
+
+template <typename Type>
+std::optional<Type> getOption(const std::unordered_map<String, Field> & options, const String & option)
+{
+    if (auto it = options.find(option); it != options.end())
     {
-        auto tokenizer = std::make_unique<NgramTokenExtractor>(ngram_length);
-        return std::make_shared<MergeTreeIndexGin>(index, gin_filter_params, std::move(tokenizer));
+        const Field & value = it->second;
+        Field::Types::Which expected_type = Field::TypeToEnum<Type>::value;
+        if (value.getType() != expected_type)
+            throw Exception(
+                ErrorCodes::INCORRECT_QUERY,
+                "GIN index argument '{}' expected to be {}, but got {}",
+                option,
+                fieldTypeToString(expected_type),
+                value.getTypeName());
+        return value.safeGet<Type>();
     }
+    return std::nullopt;
+}
+
+}
+
+MergeTreeIndexPtr ginIndexCreator(const IndexDescription & index)
+{
+    std::unordered_map<String, Field> options = convertArgumentsToOptionsMap(index.arguments);
+
+    String tokenizer = getOption<String>(options, ARGUMENT_TOKENIZER).value();
+
+    std::unique_ptr<ITokenExtractor> token_extractor;
+    if (tokenizer == SplitTokenExtractor::getExternalName())
+        token_extractor = std::make_unique<SplitTokenExtractor>();
+    else if (tokenizer == NoOpTokenExtractor::getExternalName())
+        token_extractor = std::make_unique<NoOpTokenExtractor>();
+    else if (tokenizer == NgramTokenExtractor::getExternalName())
+    {
+        UInt64 ngram_size = getOption<UInt64>(options, ARGUMENT_NGRAM_SIZE).value_or(3);
+        token_extractor = std::make_unique<NgramTokenExtractor>(ngram_size);
+    }
+    else
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Tokenizer {} not supported", tokenizer);
+
+    UInt64 max_rows_per_postings_list = getOption<UInt64>(options, ARGUMENT_MAX_ROWS).value_or(DEFAULT_MAX_ROWS_PER_POSTINGS_LIST);
+
+    GinFilterParameters params(tokenizer, max_rows_per_postings_list);
+    return std::make_shared<MergeTreeIndexGin>(index, params, std::move(token_extractor));
 }
 
 void ginIndexValidator(const IndexDescription & index, bool /*attach*/)
 {
-    /// Check number and type of arguments
-    if (index.arguments.size() > 2)
-        throw Exception(ErrorCodes::INCORRECT_QUERY, "GIN index must have less than two arguments");
+    std::unordered_map<String, Field> options = convertArgumentsToOptionsMap(index.arguments);
 
-    if (!index.arguments.empty() && index.arguments[0].getType() != Field::Types::UInt64)
-        throw Exception(ErrorCodes::INCORRECT_QUERY, "First argument of GIN index (tokenizer) must be of type UInt64");
+    /// Check that tokenizer is present and supported
+    std::optional<String> tokenizer = getOption<String>(options, ARGUMENT_TOKENIZER);
+    if (!tokenizer)
+        throw Exception(ErrorCodes::INCORRECT_QUERY, "GIN index must have an '{}' argument", ARGUMENT_TOKENIZER);
 
-    if (index.arguments.size() == 2)
+    const bool is_supported_tokenizer = (tokenizer.value() == SplitTokenExtractor::getExternalName()
+                                      || tokenizer.value() == NoOpTokenExtractor::getExternalName()
+                                      || tokenizer.value() == NgramTokenExtractor::getExternalName());
+    if (!is_supported_tokenizer)
+        throw Exception(
+            ErrorCodes::INCORRECT_QUERY,
+            "GIN index '{}' argument supports only 'default', 'ngram', and 'noop', but got {}",
+            ARGUMENT_TOKENIZER,
+            tokenizer.value());
+
+    if (tokenizer.value() == NgramTokenExtractor::getExternalName())
     {
-        if (index.arguments[1].getType() != Field::Types::UInt64)
-            throw Exception(ErrorCodes::INCORRECT_QUERY, "Second argument of GIN index (max_rows_per_postings_list) must be of type UInt64");
-        if (index.arguments[1].safeGet<UInt64>() != UNLIMITED_ROWS_PER_POSTINGS_LIST && index.arguments[1].safeGet<UInt64>() < MIN_ROWS_PER_POSTINGS_LIST)
-            throw Exception(ErrorCodes::INCORRECT_QUERY, "Second argument of GIN index (max_rows_per_postings_list) must not be less than {}", MIN_ROWS_PER_POSTINGS_LIST);
+        UInt64 ngram_size = getOption<UInt64>(options, ARGUMENT_NGRAM_SIZE).value_or(3);
+        if (ngram_size < 2 || ngram_size > 8)
+            throw Exception(
+                ErrorCodes::INCORRECT_QUERY,
+                "GIN index '{}' argument must be between 2 and 8, but got {}", ARGUMENT_NGRAM_SIZE, ngram_size);
     }
 
-    size_t ngram_length = index.arguments.empty() ? 0 : index.arguments[0].safeGet<size_t>();
-    UInt64 max_rows_per_postings_list = index.arguments.size() < 2 ? DEFAULT_MAX_ROWS_PER_POSTINGS_LIST : index.arguments[1].safeGet<UInt64>();
-    GinFilterParameters gin_filter_params(ngram_length, max_rows_per_postings_list); /// Just validate
+    /// Check that max_rows_per_postings_list is valid (if present)
+    UInt64 max_rows_per_postings_list = getOption<UInt64>(options, ARGUMENT_MAX_ROWS).value_or(DEFAULT_MAX_ROWS_PER_POSTINGS_LIST);
+    if (max_rows_per_postings_list != UNLIMITED_ROWS_PER_POSTINGS_LIST && max_rows_per_postings_list < MIN_ROWS_PER_POSTINGS_LIST)
+        throw Exception(
+            ErrorCodes::INCORRECT_QUERY,
+            "GIN index '{}' should not be less than {}", ARGUMENT_MAX_ROWS, MIN_ROWS_PER_POSTINGS_LIST);
+
+    GinFilterParameters gin_filter_params(tokenizer.value(), max_rows_per_postings_list); /// Just validate
 
     /// Check that the index is created on a single column
     if (index.column_names.size() != 1 || index.data_types.size() != 1)

tests/queries/0_stateless/02346_gin_index_bug47393.sql

@@ -7,7 +7,7 @@ CREATE TABLE tab
 (
     id UInt64,
     str String,
-    INDEX idx str TYPE gin(3) GRANULARITY 1
+    INDEX idx str TYPE gin(tokenizer = 'ngram', ngram_size = 3) GRANULARITY 1
 )
 ENGINE = MergeTree
 ORDER BY tuple()

tests/queries/0_stateless/02346_gin_index_bug52019.sql

@@ -8,7 +8,7 @@ DROP TABLE IF EXISTS tab;
 CREATE TABLE tab (
     id UInt64,
     str Map(String, String),
-    INDEX idx mapKeys(str) TYPE gin(2) GRANULARITY 1)
+    INDEX idx mapKeys(str) TYPE gin(tokenizer = 'ngram', ngram_size = 2) GRANULARITY 1)
 ENGINE = MergeTree
 ORDER BY id
 SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi',

tests/queries/0_stateless/02346_gin_index_bug54541.sql

@@ -8,7 +8,7 @@ CREATE TABLE tab
 (
     id UInt32,
     str String,
-    INDEX idx str TYPE gin
+    INDEX idx str TYPE gin(tokenizer = 'default')
 )
 ENGINE = MergeTree
 ORDER BY id

tests/queries/0_stateless/02346_gin_index_bug59039.sql

@@ -9,7 +9,7 @@ CREATE TABLE tab
 (
     id UInt64,
     doc String,
-    INDEX text_idx doc TYPE gin
+    INDEX text_idx doc TYPE gin(tokenizer = 'default')
 )
 ENGINE = MergeTree
 ORDER BY id

tests/queries/0_stateless/02346_gin_index_creation.reference

@@ -0,0 +1,13 @@
+Must not have no arguments.
+Test single tokenizer argument.
+-- tokenizer must be default, ngram or noop.
+Test ngram_size argument.
+-- ngram size must be between 2 and 8.
+Test max_rows_per_postings_list argument.
+-- max_rows_per_posting_list is set to unlimited rows.
+-- max_rows_per_posting_list should be at least 8192.
+Parameters are shuffled.
+Types are incorrect.
+Same argument appears >1 times.
+Must be created on single column.
+Must be created on String or FixedString or Array(String) or Array(FixedString) or LowCardinality(String) or LowCardinality(FixedString) columns.