Ngram config and toics optimizations

germanosin · germanosin · commit 6da326960f8b · 2025-08-21T19:20:54.000+02:00
diff --git a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java
@@ -223,6 +223,7 @@ public static class CacheProperties {
   @AllArgsConstructor
   public static class FtsProperties {
     boolean enabled = false;
+    boolean topicsNgramEnabled = false;
     int topicsMinNGram = 3;
     int topicsMaxNGram = 5;
     int filterMinNGram = 1;
diff --git a/api/src/main/java/io/kafbat/ui/service/index/ShortWordAnalyzer.java b/api/src/main/java/io/kafbat/ui/service/index/ShortWordAnalyzer.java
@@ -0,0 +1,33 @@
+package io.kafbat.ui.service.index;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+
+public class ShortWordAnalyzer extends Analyzer {
+
+  public ShortWordAnalyzer() {}
+
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName) {
+    Tokenizer tokenizer = new StandardTokenizer();
+
+    TokenStream tokenStream = new WordDelimiterGraphFilter(
+        tokenizer,
+        WordDelimiterGraphFilter.GENERATE_WORD_PARTS
+            | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE
+            | WordDelimiterGraphFilter.PRESERVE_ORIGINAL
+            | WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS
+            | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE,
+
+        null
+    );
+
+    tokenStream = new LowerCaseFilter(tokenStream);
+
+    return new TokenStreamComponents(tokenizer, tokenStream);
+  }
+}
diff --git a/api/src/main/java/io/kafbat/ui/service/index/TopicsIndex.java b/api/src/main/java/io/kafbat/ui/service/index/TopicsIndex.java
@@ -9,6 +9,7 @@
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.IntPoint;
@@ -28,10 +29,12 @@
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.QueryBuilder;
 
 public class TopicsIndex implements AutoCloseable {
   public static final String FIELD_NAME_RAW = "name_raw";
@@ -48,13 +51,24 @@ public class TopicsIndex implements AutoCloseable {
   private final Analyzer analyzer;
   private final int maxSize;
   private final ReadWriteLock closeLock = new ReentrantReadWriteLock();
+  private final boolean searchNgram;
 
   public TopicsIndex(List<InternalTopic> topics) throws IOException {
-    this(topics, 3, 5);
+    this(topics, true);
   }
 
-  public TopicsIndex(List<InternalTopic> topics, int minNgram, int maxNgram) throws IOException {
-    this.analyzer = new ShortWordNGramAnalyzer(minNgram, maxNgram);
+  public TopicsIndex(List<InternalTopic> topics, boolean ngram) throws IOException {
+    this(topics, ngram, 1, 5);
+  }
+
+  public TopicsIndex(List<InternalTopic> topics, boolean ngram, int minNgram, int maxNgram) throws IOException {
+    if (ngram) {
+      this.analyzer = new ShortWordNGramAnalyzer(minNgram, maxNgram, false);
+    } else {
+      this.analyzer = new ShortWordAnalyzer();
+    }
+
+    this.searchNgram = ngram;
     this.directory = build(topics);
     this.indexReader = DirectoryReader.open(directory);
     this.indexSearcher = new IndexSearcher(indexReader);
@@ -113,13 +127,34 @@ public List<String> find(String search, Boolean showInternal,
                            String sortField, Integer count, float minScore) throws IOException {
     closeLock.readLock().lock();
     try {
-      QueryParser queryParser = new QueryParser(FIELD_NAME, this.analyzer);
-      queryParser.setDefaultOperator(QueryParser.Operator.AND);
       Query nameQuery;
-      try {
-        nameQuery = queryParser.parse(search);
-      } catch (ParseException e) {
-        throw new RuntimeException(e);
+      if (this.searchNgram) {
+        List<String> ngrams = NgramFilter.tokenizeStringSimple(this.analyzer, search);
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (String ng : ngrams) {
+          builder.add(new TermQuery(new Term(FIELD_NAME, ng)), BooleanClause.Occur.MUST);
+        }
+        nameQuery = builder.build();
+      } else {
+        QueryParser queryParser = new QueryParser(FIELD_NAME, this.analyzer);
+        queryParser.setDefaultOperator(QueryParser.Operator.AND);
+
+        try {
+          nameQuery = queryParser.parse(search);
+
+          if (!search.contains(" ") && !search.contains("*")) {
+            String wildcardSearch = search + "*";
+            Query wildCardNameQuery = queryParser.parse(wildcardSearch);
+            BooleanQuery.Builder withWildcard = new BooleanQuery.Builder();
+            withWildcard.add(nameQuery, BooleanClause.Occur.SHOULD);
+            withWildcard.add(wildCardNameQuery, BooleanClause.Occur.SHOULD);
+            nameQuery = withWildcard.build();
+          }
+
+
+        } catch (Exception e) {
+          throw new RuntimeException(e);
+        }
       }
 
       Query internalFilter = new TermQuery(new Term(FIELD_INTERNAL, "true"));
diff --git a/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java b/api/src/main/java/io/kafbat/ui/service/metrics/scrape/ScrapedClusterState.java
@@ -216,7 +216,7 @@ private static TopicsIndex buildTopicIndex(ClustersProperties clustersProperties
       try {
         return new TopicsIndex(topicStates.values().stream().map(
             topicState -> buildInternalTopic(topicState, clustersProperties)
-        ).toList(), fts.getTopicsMinNGram(), fts.getTopicsMaxNGram());
+        ).toList(), fts.isTopicsNgramEnabled(), fts.getTopicsMinNGram(), fts.getTopicsMaxNGram());
       } catch (Exception e) {
         log.error("Error creating topics index", e);
       }
diff --git a/api/src/test/java/io/kafbat/ui/service/index/TopicsIndexTest.java b/api/src/test/java/io/kafbat/ui/service/index/TopicsIndexTest.java
@@ -1,10 +1,9 @@
 package io.kafbat.ui.service.index;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
 import io.kafbat.ui.model.InternalTopic;
 import io.kafbat.ui.model.InternalTopicConfig;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -28,7 +27,7 @@ void testFindTopicsByName() throws Exception {
                 "audit.clients.state",
                 "audit.clients.repartitioned.status",
                 "reporting.payments.by.clientId",
-                "reporting.payments.by.currencyid"
+                "reporting.payments.by.currencyId"
             )
             .map(s -> InternalTopic.builder().name(s).partitions(Map.of()).build()).toList());
 
@@ -46,17 +45,14 @@ void testFindTopicsByName() throws Exception {
         Map.entry("topic", testTopicsCount),
         Map.entry("8", 1),
         Map.entry("9", 0),
-        Map.entry("tpic", testTopicsCount),
-        Map.entry("dogs red", 1),
-        Map.entry("tpic-1", 1),
-        Map.entry("payments dlq", 1),
-        Map.entry("paymnts dlq", 1),
+        Map.entry("dog red", 1),
+        Map.entry("topic-1", 1),
+        Map.entry("payment dlq", 1),
         Map.entry("stats dlq", 0),
         Map.entry("stat", 3),
-        Map.entry("chnges", 1),
-        Map.entry("comands", 1),
-        Map.entry("id", 1),
-        Map.entry("config_retention:compact", 1)
+        Map.entry("changes", 1),
+        Map.entry("commands", 1),
+        Map.entry("id", 2)
     );
 
     SoftAssertions softly = new SoftAssertions();
@@ -68,6 +64,18 @@ void testFindTopicsByName() throws Exception {
             .isEqualTo(entry.getValue());
       }
     }
+
+    HashMap<String, Integer> indexExamples = new HashMap<>(examples);
+    indexExamples.put("config_retention:compact", 1);
+
+    try (TopicsIndex index = new TopicsIndex(topics, false)) {
+      for (Map.Entry<String, Integer> entry : indexExamples.entrySet()) {
+        List<String> resultAll = index.find(entry.getKey(), null, topics.size());
+        softly.assertThat(resultAll.size())
+            .withFailMessage("Expected %d results for '%s', but got %s", entry.getValue(), entry.getKey(), resultAll)
+            .isEqualTo(entry.getValue());
+      }
+    }
     softly.assertAll();
   }
 

Original file line number	Diff line number	Diff line change
`@@ -216,7 +216,7 @@ private static TopicsIndex buildTopicIndex(ClustersProperties clustersProperties`
`216`	`216`	`try {`
`217`	`217`	`return new TopicsIndex(topicStates.values().stream().map(`
`218`	`218`	`topicState -> buildInternalTopic(topicState, clustersProperties)`
`219`		`- ).toList(), fts.getTopicsMinNGram(), fts.getTopicsMaxNGram());`
	`219`	`+ ).toList(), fts.isTopicsNgramEnabled(), fts.getTopicsMinNGram(), fts.getTopicsMaxNGram());`
`220`	`220`	`} catch (Exception e) {`
`221`	`221`	`log.error("Error creating topics index", e);`
`222`	`222`	`}`