diff --git a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java index e9686b3bb..e521f85a9 100644 --- a/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java +++ b/api/src/main/java/io/kafbat/ui/config/ClustersProperties.java @@ -236,6 +236,7 @@ public static class CacheProperties { public static class NgramProperties { int ngramMin = 1; int ngramMax = 4; + boolean distanceScore = true; } @Data @@ -244,10 +245,10 @@ public static class NgramProperties { public static class ClusterFtsProperties { boolean enabled = true; boolean defaultEnabled = false; - NgramProperties schemas = new NgramProperties(1, 4); - NgramProperties consumers = new NgramProperties(1, 4); - NgramProperties connect = new NgramProperties(1, 4); - NgramProperties acl = new NgramProperties(1, 4); + NgramProperties schemas = new NgramProperties(1, 4, true); + NgramProperties consumers = new NgramProperties(1, 4, true); + NgramProperties connect = new NgramProperties(1, 4, true); + NgramProperties acl = new NgramProperties(1, 4, true); public boolean use(Boolean request) { if (enabled) { diff --git a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java index 4be545da9..559d9ae6c 100644 --- a/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java +++ b/api/src/main/java/io/kafbat/ui/controller/KafkaConnectController.java @@ -23,6 +23,7 @@ import io.kafbat.ui.service.mcp.McpTool; import java.util.Comparator; import java.util.Map; +import java.util.Optional; import java.util.Set; import javax.validation.Valid; import lombok.RequiredArgsConstructor; @@ -141,15 +142,18 @@ public Mono>> getAllConnectors( .operationName("getAllConnectors") .build(); + var maybeComparator = Optional.ofNullable(orderBy).map(this::getConnectorsComparator); + var comparator = sortOrder == null || sortOrder.equals(SortOrderDTO.ASC) - ? getConnectorsComparator(orderBy) - : getConnectorsComparator(orderBy).reversed(); + ? maybeComparator + : maybeComparator.map(Comparator::reversed); + + Flux connectors = kafkaConnectService.getAllConnectors(getCluster(clusterName), search, fts) + .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), clusterName)); - Flux job = kafkaConnectService.getAllConnectors(getCluster(clusterName), search, fts) - .filterWhen(dto -> accessControlService.isConnectAccessible(dto.getConnect(), clusterName)) - .sort(comparator); + Flux sorted = comparator.map(connectors::sort).orElse(connectors); - return Mono.just(ResponseEntity.ok(job)) + return Mono.just(ResponseEntity.ok(sorted)) .doOnEach(sig -> audit(context, sig)); } @@ -284,9 +288,7 @@ private Comparator getConnectorsComparator(ConnectorColumn FullConnectorInfoDTO::getName, Comparator.nullsFirst(Comparator.naturalOrder()) ); - if (orderBy == null) { - return defaultComparator; - } + return switch (orderBy) { case CONNECT -> Comparator.comparing( FullConnectorInfoDTO::getConnect, diff --git a/api/src/main/java/io/kafbat/ui/controller/SchemasController.java b/api/src/main/java/io/kafbat/ui/controller/SchemasController.java index 26c06fb21..df3aa5f2f 100644 --- a/api/src/main/java/io/kafbat/ui/controller/SchemasController.java +++ b/api/src/main/java/io/kafbat/ui/controller/SchemasController.java @@ -25,6 +25,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Optional; import javax.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -244,11 +245,15 @@ public Mono> getSchemas(String cluster List subjectsToRetrieve; boolean paginate = true; - var schemaComparator = getComparatorForSchema(orderBy); - final Comparator comparator = - sortOrder == null || !sortOrder.equals(SortOrderDTO.DESC) - ? schemaComparator : schemaComparator.reversed(); + + var schemaComparator = Optional.ofNullable(orderBy).map(this::getComparatorForSchema); + var comparator = sortOrder == null || !sortOrder.equals(SortOrderDTO.DESC) + ? schemaComparator : schemaComparator.map(Comparator::reversed); + if (orderBy == null || SchemaColumnsToSortDTO.SUBJECT.equals(orderBy)) { + if (orderBy != null) { + filteredSubjects.sort(Comparator.nullsFirst(Comparator.naturalOrder())); + } if (SortOrderDTO.DESC.equals(sortOrder)) { filteredSubjects.sort(Comparator.nullsFirst(Comparator.reverseOrder())); } @@ -274,11 +279,13 @@ public Mono> getSchemas(String cluster private List paginateSchemas( List subjects, - Comparator comparator, + Optional> comparator, boolean paginate, int pageSize, int subjectToSkip) { - subjects.sort(comparator); + + comparator.ifPresent(subjects::sort); + if (paginate) { return subjects.subList(subjectToSkip, Math.min(subjectToSkip + pageSize, subjects.size())); } else { diff --git a/api/src/main/java/io/kafbat/ui/service/index/AclBindingNgramFilter.java b/api/src/main/java/io/kafbat/ui/service/index/AclBindingNgramFilter.java index 1622e20f6..f431fa288 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/AclBindingNgramFilter.java +++ b/api/src/main/java/io/kafbat/ui/service/index/AclBindingNgramFilter.java @@ -11,7 +11,7 @@ public class AclBindingNgramFilter extends NgramFilter { private final List, AclBinding>> bindings; public AclBindingNgramFilter(Collection bindings) { - this(bindings, true, new ClustersProperties.NgramProperties(1, 4)); + this(bindings, true, new ClustersProperties.NgramProperties(1, 4, true)); } public AclBindingNgramFilter( diff --git a/api/src/main/java/io/kafbat/ui/service/index/ConsumerGroupFilter.java b/api/src/main/java/io/kafbat/ui/service/index/ConsumerGroupFilter.java index cc0ab8e03..9a34266b9 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/ConsumerGroupFilter.java +++ b/api/src/main/java/io/kafbat/ui/service/index/ConsumerGroupFilter.java @@ -11,7 +11,7 @@ public class ConsumerGroupFilter extends NgramFilter { private final List, ConsumerGroupListing>> groups; public ConsumerGroupFilter(Collection groups) { - this(groups, true, new ClustersProperties.NgramProperties(1, 4)); + this(groups, true, new ClustersProperties.NgramProperties(1, 4, true)); } public ConsumerGroupFilter( diff --git a/api/src/main/java/io/kafbat/ui/service/index/KafkaConnectNgramFilter.java b/api/src/main/java/io/kafbat/ui/service/index/KafkaConnectNgramFilter.java index 99ffd5275..16314b1bc 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/KafkaConnectNgramFilter.java +++ b/api/src/main/java/io/kafbat/ui/service/index/KafkaConnectNgramFilter.java @@ -11,7 +11,7 @@ public class KafkaConnectNgramFilter extends NgramFilter { private final List, FullConnectorInfoDTO>> connectors; public KafkaConnectNgramFilter(Collection connectors) { - this(connectors, true, new ClustersProperties.NgramProperties(1, 4)); + this(connectors, true, new ClustersProperties.NgramProperties(1, 4, true)); } public KafkaConnectNgramFilter( diff --git a/api/src/main/java/io/kafbat/ui/service/index/LuceneTopicsIndex.java b/api/src/main/java/io/kafbat/ui/service/index/LuceneTopicsIndex.java index 9d9c8bf5a..3aa708806 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/LuceneTopicsIndex.java +++ b/api/src/main/java/io/kafbat/ui/service/index/LuceneTopicsIndex.java @@ -2,6 +2,9 @@ import io.kafbat.ui.model.InternalTopic; import io.kafbat.ui.model.InternalTopicConfig; +import io.kafbat.ui.service.index.lucene.IndexedTextField; +import io.kafbat.ui.service.index.lucene.NameDistanceScoringFunction; +import io.kafbat.ui.service.index.lucene.ShortWordAnalyzer; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -18,11 +21,11 @@ import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.function.FunctionScoreQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.BooleanClause; @@ -59,11 +62,13 @@ public LuceneTopicsIndex(List topics) throws IOException { private Directory build(List topics) { Directory directory = new ByteBuffersDirectory(); + try (IndexWriter directoryWriter = new IndexWriter(directory, new IndexWriterConfig(this.analyzer))) { for (InternalTopic topic : topics) { Document doc = new Document(); + doc.add(new StringField(FIELD_NAME_RAW, topic.getName(), Field.Store.YES)); - doc.add(new TextField(FIELD_NAME, topic.getName(), Field.Store.NO)); + doc.add(new IndexedTextField(FIELD_NAME, topic.getName(), Field.Store.YES)); doc.add(new IntPoint(FIELD_PARTITIONS, topic.getPartitionCount())); doc.add(new IntPoint(FIELD_REPLICATION, topic.getReplicationFactor())); doc.add(new LongPoint(FIELD_SIZE, topic.getSegmentSize())); @@ -117,9 +122,9 @@ public List find(String search, Boolean showInternal, closeLock.readLock().lock(); try { - QueryParser queryParser = new PrefixQueryParser(FIELD_NAME, this.analyzer); + PrefixQueryParser queryParser = new PrefixQueryParser(FIELD_NAME, this.analyzer); queryParser.setDefaultOperator(QueryParser.Operator.AND); - Query nameQuery = queryParser.parse(search);; + Query nameQuery = queryParser.parse(search); Query internalFilter = new TermQuery(new Term(FIELD_INTERNAL, "true")); @@ -129,6 +134,12 @@ public List find(String search, Boolean showInternal, queryBuilder.add(internalFilter, BooleanClause.Occur.MUST_NOT); } + BooleanQuery combined = queryBuilder.build(); + Query wrapped = new FunctionScoreQuery( + combined, + new NameDistanceScoringFunction(FIELD_NAME, queryParser.getPrefixes()) + ); + List sortFields = new ArrayList<>(); sortFields.add(SortField.FIELD_SCORE); if (!sortField.equals(FIELD_NAME)) { @@ -137,7 +148,7 @@ public List find(String search, Boolean showInternal, Sort sort = new Sort(sortFields.toArray(new SortField[0])); - TopDocs result = this.indexSearcher.search(queryBuilder.build(), count != null ? count : this.maxSize, sort); + TopDocs result = this.indexSearcher.search(wrapped, count != null ? count : this.maxSize, sort); List topics = new ArrayList<>(); for (ScoreDoc scoreDoc : result.scoreDocs) { diff --git a/api/src/main/java/io/kafbat/ui/service/index/NgramFilter.java b/api/src/main/java/io/kafbat/ui/service/index/NgramFilter.java index 5afbd62a3..ab708efb3 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/NgramFilter.java +++ b/api/src/main/java/io/kafbat/ui/service/index/NgramFilter.java @@ -10,6 +10,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Stream; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -22,9 +25,11 @@ public abstract class NgramFilter { private final Analyzer analyzer; private final boolean enabled; + private final boolean distanceScore; public NgramFilter(ClustersProperties.NgramProperties properties, boolean enabled) { this.enabled = enabled; + this.distanceScore = properties.isDistanceScore(); this.analyzer = new ShortWordNGramAnalyzer(properties.getNgramMin(), properties.getNgramMax(), false); } @@ -52,15 +57,25 @@ public List find(String search, Comparator comparator) { try { List> result = new ArrayList<>(); List queryTokens = tokenizeString(analyzer, search); - Map queryFreq = termFreq(queryTokens); + Map queryFreq = Map.of(); + + if (!distanceScore) { + queryFreq = termFreq(queryTokens); + } for (Tuple2, T> item : getItems()) { for (String field : item.getT1()) { List itemTokens = tokenizeString(analyzer, field); HashSet itemTokensSet = new HashSet<>(itemTokens); if (itemTokensSet.containsAll(queryTokens)) { - double score = cosineSimilarity(queryFreq, itemTokens); + double score; + if (distanceScore) { + score = distanceSimilarity(queryTokens, itemTokens); + } else { + score = cosineSimilarity(queryFreq, itemTokens); + } result.add(new SearchResult<>(item.getT2(), score)); + break; } } } @@ -77,6 +92,22 @@ public List find(String search, Comparator comparator) { } } + private double distanceSimilarity(List queryTokens, List itemTokens) { + int smallest = Integer.MAX_VALUE; + for (String queryToken : queryTokens) { + int i = itemTokens.indexOf(queryToken); + if (i >= 0) { + smallest = Math.min(smallest, i); + } + } + + if (smallest == Integer.MAX_VALUE) { + return 1.0; + } else { + return 1.0 / (1.0 + smallest); + } + } + private List list(Stream stream, Comparator comparator) { if (comparator != null) { return stream.sorted(comparator).toList(); diff --git a/api/src/main/java/io/kafbat/ui/service/index/PrefixQueryParser.java b/api/src/main/java/io/kafbat/ui/service/index/PrefixQueryParser.java index 2db1f7133..178ba8796 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/PrefixQueryParser.java +++ b/api/src/main/java/io/kafbat/ui/service/index/PrefixQueryParser.java @@ -3,7 +3,9 @@ import static org.apache.lucene.search.BoostAttribute.DEFAULT_BOOST; import io.kafbat.ui.service.index.TopicsIndex.FieldType; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.IntPoint; @@ -14,10 +16,11 @@ import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; public class PrefixQueryParser extends QueryParser { + private final List prefixes = new ArrayList<>(); + public PrefixQueryParser(String field, Analyzer analyzer) { super(field, analyzer); } @@ -60,7 +63,13 @@ protected Query newTermQuery(Term term, float boost) { .orElse(FieldType.STRING); Query query = switch (fieldType) { - case STRING -> new PrefixQuery(term); + case STRING -> { + if (Objects.equals(term.field(), field)) { + prefixes.add(term.text()); + } + + yield new PrefixQuery(term); + } case INT -> IntPoint.newExactQuery(term.field(), Integer.parseInt(term.text())); case LONG -> LongPoint.newExactQuery(term.field(), Long.parseLong(term.text())); case BOOLEAN -> new TermQuery(term); @@ -72,4 +81,7 @@ protected Query newTermQuery(Term term, float boost) { return new BoostQuery(query, boost); } + public List getPrefixes() { + return prefixes; + } } diff --git a/api/src/main/java/io/kafbat/ui/service/index/SchemasFilter.java b/api/src/main/java/io/kafbat/ui/service/index/SchemasFilter.java index 553e63e32..3b8d1ac36 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/SchemasFilter.java +++ b/api/src/main/java/io/kafbat/ui/service/index/SchemasFilter.java @@ -16,7 +16,7 @@ public SchemasFilter(Collection subjects, boolean enabled, ClustersPrope @Override public List find(String search) { - return super.find(search, String::compareTo); + return super.find(search, null); } @Override diff --git a/api/src/main/java/io/kafbat/ui/service/index/lucene/IndexedTextField.java b/api/src/main/java/io/kafbat/ui/service/index/lucene/IndexedTextField.java new file mode 100644 index 000000000..dd29b88a3 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/index/lucene/IndexedTextField.java @@ -0,0 +1,90 @@ +package io.kafbat.ui.service.index.lucene; + + +import java.io.Reader; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.IndexOptions; + +public class IndexedTextField extends Field { + + /** Indexed, tokenized, not stored. */ + public static final FieldType TYPE_NOT_STORED = new FieldType(); + + /** Indexed, tokenized, stored. */ + public static final FieldType TYPE_STORED = new FieldType(); + + static { + TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + TYPE_NOT_STORED.setTokenized(true); + TYPE_NOT_STORED.freeze(); + + TYPE_STORED.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + TYPE_STORED.setTokenized(true); + TYPE_STORED.setStored(true); + TYPE_STORED.setStoreTermVectors(true); + TYPE_STORED.setStoreTermVectorOffsets(true); + TYPE_STORED.setStoreTermVectorPositions(true); + TYPE_STORED.freeze(); + } + + private final StoredValue storedValue; + + /** + * Creates a new un-stored TextField with Reader value. + * + * @param name field name + * @param reader reader value + * @throws IllegalArgumentException if the field name is null + * @throws NullPointerException if the reader is null + */ + public IndexedTextField(String name, Reader reader) { + super(name, reader, TYPE_NOT_STORED); + storedValue = null; + } + + /** + * Creates a new TextField with String value. + * + * @param name field name + * @param value string value + * @param store Store.YES if the content should also be stored + * @throws IllegalArgumentException if the field name or value is null. + */ + public IndexedTextField(String name, String value, Store store) { + super(name, value, store == Store.YES ? TYPE_STORED : TYPE_NOT_STORED); + if (store == Store.YES) { + storedValue = new StoredValue(value); + } else { + storedValue = null; + } + } + + /** + * Creates a new un-stored TextField with TokenStream value. + * + * @param name field name + * @param stream TokenStream value + * @throws IllegalArgumentException if the field name is null. + * @throws NullPointerException if the tokenStream is null + */ + public IndexedTextField(String name, TokenStream stream) { + super(name, stream, TYPE_NOT_STORED); + storedValue = null; + } + + @Override + public void setStringValue(String value) { + super.setStringValue(value); + if (storedValue != null) { + storedValue.setStringValue(value); + } + } + + @Override + public StoredValue storedValue() { + return storedValue; + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/index/lucene/NameDistanceScoringFunction.java b/api/src/main/java/io/kafbat/ui/service/index/lucene/NameDistanceScoringFunction.java new file mode 100644 index 000000000..e28f33424 --- /dev/null +++ b/api/src/main/java/io/kafbat/ui/service/index/lucene/NameDistanceScoringFunction.java @@ -0,0 +1,116 @@ +package io.kafbat.ui.service.index.lucene; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DoubleValues; +import org.apache.lucene.search.DoubleValuesSource; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.BytesRef; + +public class NameDistanceScoringFunction extends DoubleValuesSource { + private final String fieldName; + private final List prefixes; + + public NameDistanceScoringFunction(String fieldName, List prefixes) { + this.fieldName = fieldName; + this.prefixes = prefixes; + } + + @Override + public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException { + + Terms terms = ctx.reader().terms(fieldName); + Map positions = new HashMap<>(); + + for (String prefix : prefixes) { + TermsEnum iterator = terms.iterator(); + TermsEnum.SeekStatus seekStatus = iterator.seekCeil(new BytesRef(prefix)); + if (!seekStatus.equals(TermsEnum.SeekStatus.END)) { + + PostingsEnum postings = iterator.postings( + null, + PostingsEnum.OFFSETS | PostingsEnum.FREQS | PostingsEnum.POSITIONS + ); + + while (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + int doc = postings.docID(); + int smallest = Integer.MAX_VALUE; + + for (int i = 0; i < postings.freq(); i++) { + postings.nextPosition(); + smallest = Math.min(smallest, postings.startOffset()); + } + int finalSmall = smallest; + int s = positions.computeIfAbsent(doc, d -> finalSmall); + if (finalSmall < s) { + positions.put(doc, finalSmall); + } + } + } + } + + return new DoubleValues() { + int doc = -1; + + @Override + public double doubleValue() { + Integer pos = positions.get(doc); + if (pos == null) { + return 1.0; + } + return 1.0 / (1.0 + pos); + } + + @Override + public boolean advanceExact(int target) { + doc = target; + return true; + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + + @Override + public DoubleValuesSource rewrite(IndexSearcher searcher) { + return this; + } + + @Override + public int hashCode() { + return java.util.Objects.hash(fieldName, prefixes); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + NameDistanceScoringFunction that = (NameDistanceScoringFunction) obj; + return java.util.Objects.equals(fieldName, that.fieldName) + && java.util.Objects.equals(prefixes, that.prefixes); + } + + @Override + public String toString() { + return "NameDistanceScoringFunction"; + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; + } +} diff --git a/api/src/main/java/io/kafbat/ui/service/index/ShortWordAnalyzer.java b/api/src/main/java/io/kafbat/ui/service/index/lucene/ShortWordAnalyzer.java similarity index 81% rename from api/src/main/java/io/kafbat/ui/service/index/ShortWordAnalyzer.java rename to api/src/main/java/io/kafbat/ui/service/index/lucene/ShortWordAnalyzer.java index ee278fb09..f3e9ec589 100644 --- a/api/src/main/java/io/kafbat/ui/service/index/ShortWordAnalyzer.java +++ b/api/src/main/java/io/kafbat/ui/service/index/lucene/ShortWordAnalyzer.java @@ -1,13 +1,14 @@ -package io.kafbat.ui.service.index; +package io.kafbat.ui.service.index.lucene; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter; +import org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator; import org.apache.lucene.analysis.standard.StandardTokenizer; -class ShortWordAnalyzer extends Analyzer { +public class ShortWordAnalyzer extends Analyzer { public ShortWordAnalyzer() {} @@ -17,12 +18,13 @@ protected TokenStreamComponents createComponents(String fieldName) { TokenStream tokenStream = new WordDelimiterGraphFilter( tokenizer, + true, + WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, WordDelimiterGraphFilter.GENERATE_WORD_PARTS | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE | WordDelimiterGraphFilter.PRESERVE_ORIGINAL | WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, - null ); diff --git a/api/src/main/resources/application-localtest.yaml b/api/src/main/resources/application-localtest.yaml index 89550e4e6..d26540cbc 100644 --- a/api/src/main/resources/application-localtest.yaml +++ b/api/src/main/resources/application-localtest.yaml @@ -12,6 +12,7 @@ kafka: schemaRegistry: http://localhost:8085 fts: enabled: true + default-enabled: true dynamic.config.enabled: true diff --git a/api/src/test/java/io/kafbat/ui/service/SchemaRegistryPaginationTest.java b/api/src/test/java/io/kafbat/ui/service/SchemaRegistryPaginationTest.java index 5152c7f5d..32083848b 100644 --- a/api/src/test/java/io/kafbat/ui/service/SchemaRegistryPaginationTest.java +++ b/api/src/test/java/io/kafbat/ui/service/SchemaRegistryPaginationTest.java @@ -79,7 +79,7 @@ void shouldListFirst25andThen10Schemas() { .toList() ); var schemasFirst25 = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, - null, null, null, null, null, null, null).block(); + null, null, null, SchemaColumnsToSortDTO.SUBJECT, null, null, null).block(); assertThat(schemasFirst25).isNotNull(); assertThat(schemasFirst25.getBody()).isNotNull(); assertThat(schemasFirst25.getBody().getPageCount()).isEqualTo(4); @@ -88,7 +88,7 @@ void shouldListFirst25andThen10Schemas() { .isSortedAccordingTo(Comparator.comparing(SchemaSubjectDTO::getSubject)); var schemasFirst10 = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, - null, 10, null, null, null, null, null).block(); + null, 10, null, SchemaColumnsToSortDTO.SUBJECT, null, null, null).block(); assertThat(schemasFirst10).isNotNull(); assertThat(schemasFirst10.getBody()).isNotNull(); @@ -123,7 +123,7 @@ void shouldCorrectlyHandleNonPositivePageNumberAndPageSize() { .toList() ); var schemas = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, - 0, -1, null, null, null, null, null).block(); + 0, -1, null, SchemaColumnsToSortDTO.SUBJECT, null, null, null).block(); assertThat(schemas).isNotNull(); assertThat(schemas.getBody()).isNotNull(); @@ -142,7 +142,7 @@ void shouldCalculateCorrectPageCountForNonDivisiblePageSize() { ); var schemas = controller.getSchemas(LOCAL_KAFKA_CLUSTER_NAME, - 4, 33, null, null, null, null, null).block(); + 4, 33, null, SchemaColumnsToSortDTO.SUBJECT, null, null, null).block(); assertThat(schemas).isNotNull(); assertThat(schemas.getBody()).isNotNull(); diff --git a/api/src/test/java/io/kafbat/ui/service/index/AbstractNgramFilterTest.java b/api/src/test/java/io/kafbat/ui/service/index/AbstractNgramFilterTest.java index 801104207..ced4398c9 100644 --- a/api/src/test/java/io/kafbat/ui/service/index/AbstractNgramFilterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/index/AbstractNgramFilterTest.java @@ -6,6 +6,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -30,6 +31,16 @@ void testFind(boolean enabled) { assertThat(resultNoCompare).isNotEmpty().contains(example.getValue()); } + @Test + public void testOrder() { + List items = sortedItems(); + NgramFilter filter = buildFilter(items, true, ngramProperties); + List result = filter.find(sortedExample(items)); + assertThat(result).isEqualTo(sortedResult(items)); + } + + + protected abstract NgramFilter buildFilter(List items, boolean enabled, ClustersProperties.NgramProperties ngramProperties); @@ -39,4 +50,10 @@ protected abstract NgramFilter buildFilter(List items, protected abstract Comparator comparator(); protected abstract Map.Entry example(List items); + + protected abstract List sortedItems(); + + protected abstract String sortedExample(List items); + + protected abstract List sortedResult(List items); } diff --git a/api/src/test/java/io/kafbat/ui/service/index/AclBindingFilterTest.java b/api/src/test/java/io/kafbat/ui/service/index/AclBindingFilterTest.java index 4c54702bd..6f5cec535 100644 --- a/api/src/test/java/io/kafbat/ui/service/index/AclBindingFilterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/index/AclBindingFilterTest.java @@ -40,4 +40,45 @@ protected Map.Entry example(List items) { AclBinding binding = items.getFirst(); return Map.entry(binding.entry().principal(), binding); } + + @Override + protected List sortedItems() { + return List.of( + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "res-name-part-2", PatternType.LITERAL), + new AccessControlEntry("s2Principal", "*", AclOperation.ALL, AclPermissionType.ALLOW) + ), + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "res-name-part", PatternType.LITERAL), + new AccessControlEntry("principal-first-part", "*", AclOperation.ALL, AclPermissionType.ALLOW) + ), + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "res-name-part-2", PatternType.LITERAL), + new AccessControlEntry("s-principal", "*", AclOperation.ALL, AclPermissionType.ALLOW) + ) + ); + } + + @Override + protected String sortedExample(List items) { + return "princ"; + } + + @Override + protected List sortedResult(List items) { + return List.of( + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "res-name-part", PatternType.LITERAL), + new AccessControlEntry("principal-first-part", "*", AclOperation.ALL, AclPermissionType.ALLOW) + ), + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "res-name-part-2", PatternType.LITERAL), + new AccessControlEntry("s-principal", "*", AclOperation.ALL, AclPermissionType.ALLOW) + ), + new AclBinding( + new ResourcePattern(ResourceType.TOPIC, "res-name-part-2", PatternType.LITERAL), + new AccessControlEntry("s2Principal", "*", AclOperation.ALL, AclPermissionType.ALLOW) + ) + ); + } } diff --git a/api/src/test/java/io/kafbat/ui/service/index/ConsumerGroupFilterTest.java b/api/src/test/java/io/kafbat/ui/service/index/ConsumerGroupFilterTest.java index 0d2b73030..43e2fcdf9 100644 --- a/api/src/test/java/io/kafbat/ui/service/index/ConsumerGroupFilterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/index/ConsumerGroupFilterTest.java @@ -33,4 +33,27 @@ protected Map.Entry example(List sortedItems() { + return List.of( + new ConsumerGroupListing("cg-payment-new-", true), + new ConsumerGroupListing("payment-cg-", true), + new ConsumerGroupListing("payCg", true) + ); + } + + @Override + protected String sortedExample(List items) { + return "pay"; + } + + @Override + protected List sortedResult(List items) { + return List.of( + new ConsumerGroupListing("payment-cg-", true), + new ConsumerGroupListing("payCg", true), + new ConsumerGroupListing("cg-payment-new-", true) + ); + } } diff --git a/api/src/test/java/io/kafbat/ui/service/index/KafkaConnectNgramFilterTest.java b/api/src/test/java/io/kafbat/ui/service/index/KafkaConnectNgramFilterTest.java index 66cf7db44..2736f4dfa 100644 --- a/api/src/test/java/io/kafbat/ui/service/index/KafkaConnectNgramFilterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/index/KafkaConnectNgramFilterTest.java @@ -45,4 +45,61 @@ protected Map.Entry example(List sortedItems() { + return List.of( + new FullConnectorInfoDTO( + "connect-pay", + "connector-pay", + "class", + ConnectorTypeDTO.SINK, + List.of(), + new ConnectorStatusDTO(ConnectorStateDTO.RUNNING, "reason"), + 1, + 0 + ), + new FullConnectorInfoDTO( + "pay-connect", + "pay-connector", + "class", + ConnectorTypeDTO.SINK, + List.of(), + new ConnectorStatusDTO(ConnectorStateDTO.RUNNING, "reason"), + 1, + 0 + ) + ); + } + + @Override + protected String sortedExample(List items) { + return "pay"; + } + + @Override + protected List sortedResult(List items) { + return List.of( + new FullConnectorInfoDTO( + "pay-connect", + "pay-connector", + "class", + ConnectorTypeDTO.SINK, + List.of(), + new ConnectorStatusDTO(ConnectorStateDTO.RUNNING, "reason"), + 1, + 0 + ), + new FullConnectorInfoDTO( + "connect-pay", + "connector-pay", + "class", + ConnectorTypeDTO.SINK, + List.of(), + new ConnectorStatusDTO(ConnectorStateDTO.RUNNING, "reason"), + 1, + 0 + ) + ); + } } diff --git a/api/src/test/java/io/kafbat/ui/service/index/SchemasFilterTest.java b/api/src/test/java/io/kafbat/ui/service/index/SchemasFilterTest.java index 32d1348e3..83fa9875d 100644 --- a/api/src/test/java/io/kafbat/ui/service/index/SchemasFilterTest.java +++ b/api/src/test/java/io/kafbat/ui/service/index/SchemasFilterTest.java @@ -29,4 +29,23 @@ protected Map.Entry example(List items) { String item = items.getFirst(); return Map.entry(item, item); } + + @Override + protected List sortedItems() { + return List.of( + "longwordstartingfromPay", "s-pay", "sPayment" + ); + } + + @Override + protected String sortedExample(List items) { + return "pay"; + } + + @Override + protected List sortedResult(List items) { + return List.of( + "s-pay", "sPayment", "longwordstartingfromPay" + ); + } } diff --git a/api/src/test/java/io/kafbat/ui/service/index/LuceneTopicsIndexTest.java b/api/src/test/java/io/kafbat/ui/service/index/lucene/LuceneTopicsIndexTest.java similarity index 74% rename from api/src/test/java/io/kafbat/ui/service/index/LuceneTopicsIndexTest.java rename to api/src/test/java/io/kafbat/ui/service/index/lucene/LuceneTopicsIndexTest.java index eafa5889b..e597a3e0f 100644 --- a/api/src/test/java/io/kafbat/ui/service/index/LuceneTopicsIndexTest.java +++ b/api/src/test/java/io/kafbat/ui/service/index/lucene/LuceneTopicsIndexTest.java @@ -1,8 +1,11 @@ -package io.kafbat.ui.service.index; +package io.kafbat.ui.service.index.lucene; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import io.kafbat.ui.model.InternalPartition; import io.kafbat.ui.model.InternalTopic; import io.kafbat.ui.model.InternalTopicConfig; +import io.kafbat.ui.service.index.LuceneTopicsIndex; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -12,6 +15,9 @@ import java.util.stream.Stream; import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class LuceneTopicsIndexTest { @Test @@ -82,4 +88,24 @@ void testFindTopicsByName() throws Exception { softly.assertAll(); } + @ParameterizedTest + @MethodSource("providerOrdered") + void testOrders(List orderedTopics, String search) throws Exception { + List topics = orderedTopics.stream() + .map(s -> InternalTopic.builder().name(s).partitions(Map.of()).build()).toList(); + + + try (LuceneTopicsIndex index = new LuceneTopicsIndex(topics)) { + List resultAll = index.find(search, null, true, topics.size()); + assertThat(resultAll).isEqualTo(topics); + } + } + + public static Stream providerOrdered() { + return Stream.of( + Arguments.of(List.of("sk.long.term.name", "long.sk", "longnamebefore.sk"), "sk"), + Arguments.of(List.of("sk_long_term.name", "sk", "sk2", "long-sk", "longnamebeforeSk"), "sk") + ); + } + } diff --git a/api/src/test/java/io/kafbat/ui/service/index/lucene/ShortWordAnalyzerTest.java b/api/src/test/java/io/kafbat/ui/service/index/lucene/ShortWordAnalyzerTest.java new file mode 100644 index 000000000..9113ff5ba --- /dev/null +++ b/api/src/test/java/io/kafbat/ui/service/index/lucene/ShortWordAnalyzerTest.java @@ -0,0 +1,56 @@ +package io.kafbat.ui.service.index.lucene; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.util.function.Tuple3; +import reactor.util.function.Tuples; + +class ShortWordAnalyzerTest { + + @ParameterizedTest + @MethodSource("provider") + public void testOffsets(String name, List> parts) throws Exception { + ShortWordAnalyzer analyzer = new ShortWordAnalyzer(); + + TokenStream ts = analyzer.tokenStream("content", new StringReader(name)); + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + OffsetAttribute offAtt = ts.addAttribute(OffsetAttribute.class); + ts.reset(); + + List> calculated = new ArrayList<>(); + while (ts.incrementToken()) { + calculated.add(Tuples.of(termAtt.toString(), offAtt.startOffset(), offAtt.endOffset())); + } + ts.end(); + ts.close(); + + assertThat(calculated).isEqualTo(parts); + } + + public static Stream provider() { + return Stream.of( + Arguments.of("hello.world.text", List.of( + Tuples.of("hello.world.text", 0, 16), + Tuples.of("hello", 0, 5), + Tuples.of("world", 6, 11), + Tuples.of("text", 12, 16) + )), + Arguments.of("helloWorldText", List.of( + Tuples.of("helloworldtext", 0, 14), + Tuples.of("hello", 0, 5), + Tuples.of("world", 5, 10), + Tuples.of("text", 10, 14) + )) + ); + } +}