feat(search): FT.SEARCH FILTER option (#5830)

Even it is deprecated FILTER can still be used as part of FT.SEARCH
command. Add support for it by reconstruction of AST tree.

Closes #5822

Signed-off-by: mkaruza <[email protected]>

Author: mkaruza

src/core/search/base.h

@@ -38,6 +38,19 @@ struct QueryParams {
   absl::flat_hash_map<std::string, std::string> params;
 };
 
+// Base class for optional search filters
+
+class AstNode;
+
+struct OptionalFilterBase {
+  virtual bool IsEmpty() const = 0;
+  virtual AstNode Node(std::string field) = 0;
+  virtual ~OptionalFilterBase() = default;
+};
+
+using OptionalFilters =
+    absl::flat_hash_map<std::string /*field*/, std::unique_ptr<OptionalFilterBase> /* filter */>;
+
 // Values are either sortable as doubles or strings, or not sortable at all.
 using SortableValue = std::variant<std::monostate, double, std::string>;
 

src/core/search/query_driver.cc

@@ -22,6 +22,14 @@ void QueryDriver::Error(const Parser::location_type& loc, std::string_view msg)
   VLOG(1) << "Parse error " << loc << ": " << msg;
 }
 
+void QueryDriver::SetOptionalFilters(const OptionalFilters* filters) {
+  if (filters) {
+    for (auto& [field, filter] : *filters) {
+      expr_ = AstLogicalNode(std::move(expr_), filter->Node(field), AstLogicalNode::AND);
+    }
+  }
+}
+
 }  // namespace search
 
 }  // namespace dfly

src/core/search/query_driver.h

@@ -30,6 +30,8 @@ class QueryDriver {
     scanner_->SetParams(params);
   }
 
+  void SetOptionalFilters(const OptionalFilters* filters);
+
   Parser::symbol_type Lex() {
     return scanner()->Lex();
   }

src/core/search/search.cc

@@ -31,12 +31,14 @@ namespace dfly::search {
 
 namespace {
 
-AstExpr ParseQuery(std::string_view query, const QueryParams* params) {
+AstExpr ParseQuery(std::string_view query, const QueryParams* params,
+                   const OptionalFilters* filters) {
   QueryDriver driver{};
   driver.ResetScanner();
   driver.SetParams(params);
   driver.SetInput(std::string{query});
   (void)Parser (&driver)();  // can throw
+  driver.SetOptionalFilters(filters);
   return driver.Take();
 }
 
@@ -430,6 +432,10 @@ struct BasicSearch {
 
 }  // namespace
 
+AstNode OptionalNumericFilter::Node(std::string field) {
+  return AstFieldNode{"@" + field, AstRangeNode(lo_, false, hi_, false)};
+}
+
 string_view Schema::LookupAlias(string_view alias) const {
   if (auto it = field_names.find(alias); it != field_names.end())
     return it->second;
@@ -611,9 +617,10 @@ const Synonyms* FieldIndices::GetSynonyms() const {
 SearchAlgorithm::SearchAlgorithm() = default;
 SearchAlgorithm::~SearchAlgorithm() = default;
 
-bool SearchAlgorithm::Init(string_view query, const QueryParams* params) {
+bool SearchAlgorithm::Init(string_view query, const QueryParams* params,
+                           const OptionalFilters* filters) {
   try {
-    query_ = make_unique<AstExpr>(ParseQuery(query, params));
+    query_ = make_unique<AstExpr>(ParseQuery(query, params, filters));
   } catch (const Parser::syntax_error& se) {
     LOG(INFO) << "Failed to parse query \"" << query << "\":" << se.what();
     return false;

src/core/search/search.h

@@ -24,6 +24,35 @@ namespace dfly::search {
 struct AstNode;
 struct TextIndex;
 
+// Optional FILTER
+struct OptionalNumericFilter : public OptionalFilterBase {
+  OptionalNumericFilter(size_t lo, size_t hi) : empty_(false), lo_(lo), hi_(hi) {
+  }
+
+  bool IsEmpty() const override {
+    return empty_;
+  }
+
+  AstNode Node(std::string field) override;
+
+  void AddRange(size_t lo, size_t hi) {
+    if (empty_) {
+      return;
+    }
+    if ((hi_ < lo) || (hi < lo_)) {
+      empty_ = true;
+    } else {
+      lo_ = std::max(lo_, lo);
+      hi_ = std::min(hi_, hi);
+    }
+  }
+
+ private:
+  bool empty_;
+  size_t lo_;
+  size_t hi_;
+};
+
 // Describes a specific index field
 struct SchemaField {
   enum FieldType { TAG, TEXT, NUMERIC, VECTOR };
@@ -164,8 +193,9 @@ class SearchAlgorithm {
   SearchAlgorithm();
   ~SearchAlgorithm();
 
-  // Init with query and return true if successful.
-  bool Init(std::string_view query, const QueryParams* params);
+  // Init with query and optional filters and return true if successful.
+  bool Init(std::string_view query, const QueryParams* params,
+            const OptionalFilters* filters = nullptr);
 
   SearchResult Search(const FieldIndices* index) const;
 

src/server/search/doc_index.h

@@ -113,6 +113,9 @@ struct SearchParams {
   std::optional<std::vector<FieldReference>> load_fields;
 
   std::optional<SortOption> sort_option;
+
+  search::OptionalFilters optional_filters;
+
   search::QueryParams query_params;
 
   bool ShouldReturnAllFields() const {

src/server/search/search_family.cc

@@ -344,6 +344,20 @@ std::optional<std::string_view> ParseFieldWithAtSign(CmdArgParser* parser) {
   return field;
 }
 
+void ParseNumericFilter(CmdArgParser* parser, SearchParams* params) {
+  auto field = ParseField(parser);
+  size_t lo = parser->Next<size_t>();
+  size_t hi = parser->Next<size_t>();
+  if (auto it = params->optional_filters.find(field); it != params->optional_filters.end()) {
+    search::OptionalNumericFilter* numeric_filter =
+        dynamic_cast<search::OptionalNumericFilter*>(it->second.get());
+    numeric_filter->AddRange(lo, hi);
+  } else {
+    params->optional_filters.emplace(field,
+                                     std::make_unique<search::OptionalNumericFilter>(lo, hi));
+  }
+}
+
 std::vector<FieldReference> ParseLoadOrReturnFields(CmdArgParser* parser, bool is_load) {
   // TODO: Change to num_strings. In Redis strings number is expected. For example: LOAD 3 $.a AS a
   std::vector<FieldReference> fields;
@@ -396,6 +410,8 @@ ParseResult<SearchParams> ParseSearchParams(CmdArgParser* parser) {
       FieldReference field{ParseField(parser)};
       params.sort_option =
           SearchParams::SortOption{field, parser->Check("DESC") ? SortOrder::DESC : SortOrder::ASC};
+    } else if (parser->Check("FILTER")) {
+      ParseNumericFilter(parser, &params);
     } else {
       // Unsupported parameters are ignored for now
       parser->Skip(1);
@@ -1214,7 +1230,7 @@ void SearchFamily::FtSearch(CmdArgList args, const CommandContext& cmd_cntx) {
     return;
 
   search::SearchAlgorithm search_algo;
-  if (!search_algo.Init(query_str, &params->query_params))
+  if (!search_algo.Init(query_str, &params->query_params, &params->optional_filters))
     return builder->SendError("Query syntax error");
 
   // Because our coordinator thread may not have a shard, we can't check ahead if the index exists.

src/server/search/search_family_test.cc

@@ -3203,4 +3203,47 @@ TEST_F(SearchFamilyTest, AggregateWithLoadFromSortBySeveralFields) {
   EXPECT_THAT(resp.GetVec(), ElementsAreArray(matchers));
 }
 
+TEST_F(SearchFamilyTest, NumericFilter) {
+  // Index name, age, height
+  Run({"FT.CREATE", "i1", "ON", "HASH", "SCHEMA", "name", "TEXT", "age", "NUMERIC", "height",
+       "NUMERIC"});
+
+  // Index name, age
+  Run({"FT.CREATE", "i2", "ON", "HASH", "SCHEMA", "name", "TEXT", "age", "NUMERIC"});
+
+  Run({"HSET", "id:1", "name", "John", "age", "28", "height", "184"});
+  Run({"HSET", "id:2", "name", "Ivan", "age", "30", "height", "180"});
+  Run({"HSET", "id:3", "name", "Jon", "age", "25", "height", "182"});
+  Run({"HSET", "id:4", "name", "Juan", "age", "32", "height", "186"});
+  Run({"HSET", "id:5", "name", "Ioan", "age", "35", "height", "181"});
+
+  // Filter with non-star query
+  auto res = Run({"FT.SEARCH", "i1", "I*", "FILTER", "age", "31", "40"});
+  EXPECT_THAT(res, AreDocIds("id:5"));
+
+  // Filter on ONE NUMERIC index
+  res = Run({"FT.SEARCH", "i1", "*", "FILTER", "age", "25", "28"});
+  EXPECT_THAT(res, AreDocIds("id:1", "id:3"));
+
+  // Filter on TWO NUMERIC indexes
+  res =
+      Run({"FT.SEARCH", "i1", "*", "FILTER", "age", "25", "28", "FILTER", "height", "180", "182"});
+  EXPECT_THAT(res, AreDocIds("id:3"));
+
+  // Filter on TWO NUMERIC indexes where second filtering produce empty result
+  res =
+      Run({"FT.SEARCH", "i1", "*", "FILTER", "age", "25", "28", "FILTER", "height", "200", "300"});
+  EXPECT_THAT(res, AreDocIds());
+
+  // Filter on index which doesn't exists
+  res = Run({"FT.SEARCH", "i2", "*", "FILTER", "height", "180", "190"});
+  EXPECT_THAT(res, ErrArg("Invalid field: height"));
+
+  // Two filters on same field
+  res = Run({"FT.SEARCH", "i1", "J*", "FILTER", "age", "25", "30", "FILTER", "age", "28", "32"});
+  EXPECT_THAT(res, AreDocIds("id:1"));
+
+  Run({"FLUSHALL"});
+}
+
 }  // namespace dfly