diff --git a/build.xml b/build.xml index ff02dcbdf889..ceb5c436ee87 100644 --- a/build.xml +++ b/build.xml @@ -1723,7 +1723,6 @@ - diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java index 7381c145c007..bdbb932a791a 100644 --- a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java @@ -729,11 +729,11 @@ long currentOperations() private DecoratedKey firstPartitionKey(Direction direction) { - Iterator> iter = data.filteredEntryIterator(direction, PartitionData.class); + Iterator> iter = data.filteredEntryIterator(direction, PartitionData.class); if (!iter.hasNext()) return null; - Map.Entry entry = iter.next(); + Map.Entry entry = iter.next(); return getPartitionKeyFromPath(metadata.get(), entry.getKey()); } diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java index 708ce85f2bf1..5105144e055e 100644 --- a/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java +++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtableStage1.java @@ -404,7 +404,7 @@ private static MemtablePartition createPartition(TableMetadata metadata, EnsureO return new MemtablePartition(metadata, ensureOnHeap, key, data); } - private static MemtablePartition getPartitionFromTrieEntry(TableMetadata metadata, EnsureOnHeap ensureOnHeap, Map.Entry en) + private static MemtablePartition getPartitionFromTrieEntry(TableMetadata metadata, EnsureOnHeap ensureOnHeap, Map.Entry en) { DecoratedKey key = BufferDecoratedKey.fromByteComparable(en.getKey(), BYTE_COMPARABLE_VERSION, @@ -423,10 +423,11 @@ public FlushCollection getFlushSet(PartitionPosition from, Pa long keySize = 0; int keyCount = 0; - for (Iterator> it = toFlush.entryIterator(); it.hasNext(); ) + for (Iterator> it = toFlush.entryIterator(); it.hasNext(); ) { - Map.Entry en = it.next(); - byte[] keyBytes = DecoratedKey.keyFromByteComparable(en.getKey(), BYTE_COMPARABLE_VERSION, metadata().partitioner); + Map.Entry en = it.next(); + ByteComparable byteComparable = v -> en.getKey().asPeekableBytes(BYTE_COMPARABLE_VERSION); + byte[] keyBytes = DecoratedKey.keyFromByteComparable(byteComparable, BYTE_COMPARABLE_VERSION, metadata().partitioner); keySize += keyBytes.length; keyCount++; } @@ -628,11 +629,11 @@ long currentOperations() private DecoratedKey firstPartitionKey(Direction direction) { - Iterator> iter = data.entryIterator(direction); + Iterator> iter = data.entryIterator(direction); if (!iter.hasNext()) return null; - Map.Entry entry = iter.next(); + Map.Entry entry = iter.next(); return getPartitionKeyFromPath(metadata.get(), entry.getKey()); } @@ -652,7 +653,7 @@ static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredParti private final TableMetadata metadata; private final EnsureOnHeap ensureOnHeap; private final Trie source; - private final Iterator> iter; + private final Iterator> iter; private final ColumnFilter columnFilter; private final DataRange dataRange; diff --git a/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java b/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java index ecddfd9544ef..ec2578dcea90 100644 --- a/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java +++ b/src/java/org/apache/cassandra/db/tries/InMemoryReadTrie.java @@ -1132,11 +1132,6 @@ public boolean isEmpty() return isNull(root); } - public ByteComparable.Version byteComparableVersion() - { - return byteComparableVersion; - } - /** * Override of dump to provide more detailed printout that includes the type of each node in the trie. * We do this via a wrapping cursor that returns a content string for the type of node for every node we return. diff --git a/src/java/org/apache/cassandra/db/tries/Trie.java b/src/java/org/apache/cassandra/db/tries/Trie.java index dbe7e26699c8..7125938e06e5 100644 --- a/src/java/org/apache/cassandra/db/tries/Trie.java +++ b/src/java/org/apache/cassandra/db/tries/Trie.java @@ -375,7 +375,7 @@ public void forEachValue(ValueConsumer consumer) /** * Call the given consumer on all (path, content) pairs with non-null content in the trie in order. */ - public void forEachEntry(BiConsumer consumer) + public void forEachEntry(BiConsumer consumer) { forEachEntry(Direction.FORWARD, consumer); } @@ -383,7 +383,7 @@ public void forEachEntry(BiConsumer consumer) /** * Call the given consumer on all (path, content) pairs with non-null content in the trie in order. */ - public void forEachEntry(Direction direction, BiConsumer consumer) + public void forEachEntry(Direction direction, BiConsumer consumer) { Cursor cursor = cursor(direction); process(new TrieEntriesWalker.WithConsumer(consumer, cursor.byteComparableVersion()), cursor); @@ -427,7 +427,7 @@ public Void forEachValueSkippingBranches(Direction direction, ValueConsumer c * Call the given consumer on all (path, content) pairs with non-null content in the trie in order, skipping all * branches below the top content-bearing node. */ - public void forEachEntrySkippingBranches(Direction direction, BiConsumer consumer) + public void forEachEntrySkippingBranches(Direction direction, BiConsumer consumer) { Cursor cursor = cursor(direction); processSkippingBranches(new TrieEntriesWalker.WithConsumer(consumer, cursor.byteComparableVersion()), cursor); @@ -545,7 +545,7 @@ public Trie subtrie(ByteComparable left, ByteComparable right) /** * Returns the ordered entry set of this trie's content as an iterable. */ - public Iterable> entrySet() + public Iterable> entrySet() { return this::entryIterator; } @@ -553,7 +553,7 @@ public Iterable> entrySet() /** * Returns the ordered entry set of this trie's content as an iterable. */ - public Iterable> entrySet(Direction direction) + public Iterable> entrySet(Direction direction) { return () -> entryIterator(direction); } @@ -561,7 +561,7 @@ public Iterable> entrySet(Direction dire /** * Returns the ordered entry set of this trie's content in an iterator. */ - public Iterator> entryIterator() + public Iterator> entryIterator() { return entryIterator(Direction.FORWARD); } @@ -569,7 +569,7 @@ public Iterator> entryIterator() /** * Returns the ordered entry set of this trie's content in an iterator. */ - public Iterator> entryIterator(Direction direction) + public Iterator> entryIterator(Direction direction) { return new TrieEntriesIterator.AsEntries<>(cursor(direction)); } @@ -577,7 +577,7 @@ public Iterator> entryIterator(Direction /** * Returns the ordered entry set of this trie's content in an iterable, filtered by the given type. */ - public Iterable> filteredEntrySet(Class clazz) + public Iterable> filteredEntrySet(Class clazz) { return filteredEntrySet(Direction.FORWARD, clazz); } @@ -585,7 +585,7 @@ public Iterable> filteredE /** * Returns the ordered entry set of this trie's content in an iterable, filtered by the given type. */ - public Iterable> filteredEntrySet(Direction direction, Class clazz) + public Iterable> filteredEntrySet(Direction direction, Class clazz) { return () -> filteredEntryIterator(direction, clazz); } @@ -593,7 +593,7 @@ public Iterable> filteredE /** * Returns the ordered entry set of this trie's content in an iterator, filtered by the given type. */ - public Iterator> filteredEntryIterator(Direction direction, Class clazz) + public Iterator> filteredEntryIterator(Direction direction, Class clazz) { return new TrieEntriesIterator.AsEntriesFilteredByType<>(cursor(direction), clazz); } @@ -801,7 +801,7 @@ public Trie prefixedBy(ByteComparable prefix) * Returns an entry set containing all tail tree constructed at the points that contain content of * the given type. */ - public Iterable>> tailTries(Direction direction, Class clazz) + public Iterable>> tailTries(Direction direction, Class clazz) { return () -> new TrieTailsIterator.AsEntries<>(cursor(direction), clazz); } diff --git a/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java b/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java index 99e3f764244d..c4ef97b2587f 100644 --- a/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java +++ b/src/java/org/apache/cassandra/db/tries/TrieEntriesIterator.java @@ -86,7 +86,7 @@ ByteComparable.Version byteComparableVersion() /** * Iterator representing the content of the trie a sequence of (path, content) pairs. */ - static class AsEntries extends TrieEntriesIterator> + static class AsEntries extends TrieEntriesIterator> { public AsEntries(Trie.Cursor cursor) { @@ -94,7 +94,7 @@ public AsEntries(Trie.Cursor cursor) } @Override - protected Map.Entry mapContent(T content, byte[] bytes, int byteLength) + protected Map.Entry mapContent(T content, byte[] bytes, int byteLength) { return toEntry(byteComparableVersion(), content, bytes, byteLength); } @@ -103,7 +103,7 @@ protected Map.Entry mapContent(T content, byte[] b /** * Iterator representing the content of the trie a sequence of (path, content) pairs. */ - static class AsEntriesFilteredByType extends TrieEntriesIterator> + static class AsEntriesFilteredByType extends TrieEntriesIterator> { public AsEntriesFilteredByType(Trie.Cursor cursor, Class clazz) { @@ -112,13 +112,13 @@ public AsEntriesFilteredByType(Trie.Cursor cursor, Class clazz) @Override @SuppressWarnings("unchecked") // checked by the predicate - protected Map.Entry mapContent(T content, byte[] bytes, int byteLength) + protected Map.Entry mapContent(T content, byte[] bytes, int byteLength) { return toEntry(byteComparableVersion(), (U) content, bytes, byteLength); } } - static java.util.Map.Entry toEntry(ByteComparable.Version version, T content, byte[] bytes, int byteLength) + static java.util.Map.Entry toEntry(ByteComparable.Version version, T content, byte[] bytes, int byteLength) { return new AbstractMap.SimpleImmutableEntry<>(toByteComparable(version, bytes, byteLength), content); } diff --git a/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java b/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java index 362fe8f112b7..e6cf39ad121a 100644 --- a/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java +++ b/src/java/org/apache/cassandra/db/tries/TrieEntriesWalker.java @@ -40,10 +40,10 @@ public void content(T content) */ static class WithConsumer extends TrieEntriesWalker { - private final BiConsumer consumer; + private final BiConsumer consumer; private final ByteComparable.Version byteComparableVersion; - public WithConsumer(BiConsumer consumer, ByteComparable.Version byteComparableVersion) + public WithConsumer(BiConsumer consumer, ByteComparable.Version byteComparableVersion) { this.consumer = consumer; this.byteComparableVersion = byteComparableVersion; diff --git a/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java b/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java index c59d126fe272..b1ec6bead09f 100644 --- a/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java +++ b/src/java/org/apache/cassandra/db/tries/TriePathReconstructor.java @@ -49,7 +49,7 @@ public void resetPathLength(int newLength) keyPos = newLength; } - static ByteComparable.Preencoded toByteComparable(ByteComparable.Version byteComparableVersion, byte[] bytes, int byteLength) + static ByteComparable toByteComparable(ByteComparable.Version byteComparableVersion, byte[] bytes, int byteLength) { // Taking a copy here to make sure it does not get modified when the cursor advances. return ByteComparable.preencoded(byteComparableVersion, Arrays.copyOf(bytes, byteLength)); diff --git a/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java b/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java index e15ce6548206..503be6bd359a 100644 --- a/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java +++ b/src/java/org/apache/cassandra/db/tries/TrieTailsIterator.java @@ -101,7 +101,7 @@ ByteComparable.Version byteComparableVersion() * {@code tail} is the branch of the trie rooted at the selected content node (reachable by following * {@code path}). The tail trie will have the selected content at its root. */ - static class AsEntries extends TrieTailsIterator>> + static class AsEntries extends TrieTailsIterator>> { public AsEntries(Trie.Cursor cursor, Class clazz) { @@ -109,9 +109,9 @@ public AsEntries(Trie.Cursor cursor, Class clazz) } @Override - protected Map.Entry> mapContent(T value, Trie tailTrie, byte[] bytes, int byteLength) + protected Map.Entry> mapContent(T value, Trie tailTrie, byte[] bytes, int byteLength) { - ByteComparable.Preencoded key = toByteComparable(byteComparableVersion(), bytes, byteLength); + ByteComparable key = toByteComparable(byteComparableVersion(), bytes, byteLength); return new AbstractMap.SimpleImmutableEntry<>(key, tailTrie); } } diff --git a/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java b/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java index 75f37519a966..615f3cfb7a4e 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java +++ b/src/java/org/apache/cassandra/index/sai/disk/MemtableTermsIterator.java @@ -34,16 +34,16 @@ public class MemtableTermsIterator implements TermsIterator { private final ByteBuffer minTerm; private final ByteBuffer maxTerm; - private final Iterator>> iterator; + private final Iterator>> iterator; - private Pair> current; + private Pair> current; private int maxSSTableRowId = -1; private int minSSTableRowId = Integer.MAX_VALUE; public MemtableTermsIterator(ByteBuffer minTerm, ByteBuffer maxTerm, - Iterator>> iterator) + Iterator>> iterator) { Preconditions.checkArgument(iterator != null); this.minTerm = minTerm; diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java index 266100a76789..8683409896ad 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java @@ -157,14 +157,14 @@ private long flush(DecoratedKey minKey, DecoratedKey maxKey, AbstractType ter var docLengths = new Int2IntHashMap(Integer.MIN_VALUE); Arrays.stream(((TrieMemtableIndex) memtableIndex).getRangeIndexes()) .map(TrieMemoryIndex.class::cast) - .forEach(trieMemoryIndex -> + .forEach(trieMemoryIndex -> trieMemoryIndex.getDocLengths().forEach((pk, length) -> { int rowId = rowMapping.get(pk); if (rowId >= 0) docLengths.put(rowId, (int) length); }) ); - + indexMetas = writer.writeAll(metadataBuilder.intercept(terms), docLengths); numRows = writer.getPostingsCount(); } diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java index eec15eb53589..bfc14c2f3125 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SegmentMetadataBuilder.java @@ -33,7 +33,6 @@ import org.apache.cassandra.index.sai.IndexContext; import org.apache.cassandra.index.sai.disk.PostingList; import org.apache.cassandra.index.sai.disk.TermsIterator; -import org.apache.cassandra.index.sai.disk.format.IndexComponentType; import org.apache.cassandra.index.sai.disk.format.IndexComponents; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.disk.v1.kdtree.MutableOneDimPointValues; @@ -58,7 +57,6 @@ public class SegmentMetadataBuilder private final long segmentRowIdOffset; private final List interceptors = new ArrayList<>(); - private final ByteComparable.Version byteComparableVersion; private boolean built = false; @@ -81,11 +79,10 @@ public SegmentMetadataBuilder(long segmentRowIdOffset, IndexComponents.ForWrite { IndexContext context = Objects.requireNonNull(components.context()); this.segmentRowIdOffset = segmentRowIdOffset; - this.byteComparableVersion = components.byteComparableVersionFor(IndexComponentType.TERMS_DATA); int histogramSize = context.getIntOption(HISTOGRAM_SIZE_OPTION, 128); int mostFrequentTermsCount = context.getIntOption(MFT_COUNT_OPTION, 128); - this.termsDistributionBuilder = new TermsDistribution.Builder(context.getValidator(), byteComparableVersion, histogramSize, mostFrequentTermsCount); + this.termsDistributionBuilder = new TermsDistribution.Builder(context.getValidator(), histogramSize, mostFrequentTermsCount); } public void setKeyRange(@Nonnull PrimaryKey minKey, @Nonnull PrimaryKey maxKey) @@ -350,7 +347,7 @@ public void intersect(IntersectVisitor visitor) throws IOException if (!Arrays.equals(term, lastTerm)) { if (lastTerm != null) - builder.add(ByteComparable.preencoded(builder.byteComparableVersion, lastTerm), count); + builder.add(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, lastTerm), count); count = 0; @@ -366,7 +363,7 @@ public void close() throws IOException { if (lastTerm != null) { - builder.add(ByteComparable.preencoded(builder.byteComparableVersion, lastTerm), count); + builder.add(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, lastTerm), count); } } diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java index 0a81fb5260d4..2f0f770ccc03 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValues.java @@ -24,7 +24,6 @@ import org.apache.cassandra.index.sai.disk.TermsIterator; import org.apache.cassandra.index.sai.disk.oldlucene.MutablePointValues; import org.apache.cassandra.index.sai.utils.TypeUtil; -import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; import org.apache.lucene.util.bkd.BKDWriter; @@ -56,7 +55,7 @@ public void intersect(IntersectVisitor visitor) throws IOException { while (termEnum.hasNext()) { - ByteSourceInverse.readBytesMustFit(((ByteComparable.Preencoded) termEnum.next()).getPreencodedBytes(), + ByteSourceInverse.readBytesMustFit(termEnum.next().asComparableBytes(TypeUtil.BYTE_COMPARABLE_VERSION), scratch); try (final PostingList postings = termEnum.postings()) diff --git a/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java b/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java index 9222b89f8230..2243de8af74e 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java +++ b/src/java/org/apache/cassandra/index/sai/disk/v6/TermsDistribution.java @@ -440,7 +440,6 @@ private static Version decodeIndexVersion(String versionStr) throws IOException public static class Builder { final AbstractType termType; - final ByteComparable.Version byteComparableVersion; final int histogramSize; final int mostFrequentTermsTableSize; @@ -454,12 +453,10 @@ public static class Builder long cumulativeRowCount; public Builder(AbstractType termType, - ByteComparable.Version byteComparableVersion, int histogramSize, int mostFrequentTermsTableSize) { this.termType = termType; - this.byteComparableVersion = byteComparableVersion; this.histogramSize = histogramSize; this.mostFrequentTermsTableSize = mostFrequentTermsTableSize; @@ -503,12 +500,13 @@ public TermsDistribution build() shrink(); - var mft = new TreeMap((b1, b2) -> ByteComparable.compare(b1, b2, byteComparableVersion)); + var bcVersion = TypeUtil.BYTE_COMPARABLE_VERSION; + var mft = new TreeMap((b1, b2) -> ByteComparable.compare(b1, b2, bcVersion)); for (Point point : mostFrequentTerms) { mft.put(point.term, point.rowCount); } - return new TermsDistribution(termType, buckets, mft, Version.latest(), byteComparableVersion); + return new TermsDistribution(termType, buckets, mft, Version.latest(), bcVersion); } /** diff --git a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java index 7dd696e05cbd..43cd4999c6d3 100644 --- a/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java +++ b/src/java/org/apache/cassandra/index/sai/disk/vector/VectorMemtableIndex.java @@ -432,7 +432,7 @@ public static int ensureSaneEstimate(int rawEstimate, int rerankK, int graphSize } @Override - public Iterator>> iterator(DecoratedKey min, DecoratedKey max) + public Iterator>> iterator(DecoratedKey min, DecoratedKey max) { // This method is only used when merging an in-memory index with a RowMapping. This is done a different // way with the graph using the writeData method below. diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java index 4f1fe7724198..5ffb302522bf 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/MemoryIndex.java @@ -87,7 +87,7 @@ public abstract void update(DecoratedKey key, /** * Iterate all Term->PrimaryKeys mappings in sorted order */ - public abstract Iterator>> iterator(); + public abstract Iterator>> iterator(); public static class PkWithFrequency { diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java index f2d55017e818..d349e80dceda 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java @@ -77,7 +77,7 @@ public interface MemtableIndex extends MemtableOrdering long estimateMatchingRowsCount(Expression expression, AbstractBounds keyRange); - Iterator>> iterator(DecoratedKey min, DecoratedKey max); + Iterator>> iterator(DecoratedKey min, DecoratedKey max); static MemtableIndex createIndex(IndexContext indexContext, Memtable mt) { diff --git a/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java b/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java index f97928cc93c8..e0cdc610a63c 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java +++ b/src/java/org/apache/cassandra/index/sai/memory/RowMapping.java @@ -45,7 +45,7 @@ public class RowMapping public static final RowMapping DUMMY = new RowMapping() { @Override - public Iterator>> merge(MemtableIndex index) { return Collections.emptyIterator(); } + public Iterator>> merge(MemtableIndex index) { return Collections.emptyIterator(); } @Override public void complete() {} @@ -99,7 +99,7 @@ public RowIdWithFrequency(int rowId, int frequency) { this.frequency = frequency; } } - + /** * Merge IndexMemtable(index term to PrimaryKeys mappings) with row mapping of a sstable * (PrimaryKey to RowId mappings). @@ -108,7 +108,7 @@ public RowIdWithFrequency(int rowId, int frequency) { * * @return iterator of index term to postings mapping exists in the sstable */ - public Iterator>> merge(MemtableIndex index) + public Iterator>> merge(MemtableIndex index) { assert complete : "RowMapping is not built."; @@ -116,7 +116,7 @@ public Iterator>> merge return new AbstractGuavaIterator<>() { @Override - protected Pair> computeNext() + protected Pair> computeNext() { while (it.hasNext()) { diff --git a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java index 8340a1b2fc8c..62ad188c1259 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java @@ -119,7 +119,7 @@ public TrieMemoryIndex(IndexContext indexContext, Memtable memtable, AbstractBou this.primaryKeysAccumulator = new PrimaryKeysAccumulator(primaryKeysHeapAllocations); this.primaryKeysRemover = new PrimaryKeysRemover(primaryKeysHeapAllocations); this.analyzerTransformsValue = indexContext.getAnalyzerFactory().create().transformValue(); - this.data = InMemoryTrie.longLived(TypeUtil.byteComparableVersionForTermsData(), TrieMemtable.BUFFER_TYPE, indexContext.columnFamilyStore().readOrdering()); + this.data = InMemoryTrie.longLived(TypeUtil.BYTE_COMPARABLE_VERSION, TrieMemtable.BUFFER_TYPE, indexContext.columnFamilyStore().readOrdering()); this.memtable = memtable; } @@ -273,9 +273,9 @@ private void applyTransformer(PrimaryKey primaryKey, } @Override - public Iterator>> iterator() + public Iterator>> iterator() { - Iterator> iterator = data.entrySet().iterator(); + Iterator> iterator = data.entrySet().iterator(); return new Iterator<>() { @Override @@ -285,9 +285,9 @@ public boolean hasNext() } @Override - public Pair> next() + public Pair> next() { - Map.Entry entry = iterator.next(); + Map.Entry entry = iterator.next(); var pairs = new ArrayList(entry.getValue().size()); Iterators.addAll(pairs, entry.getValue().iterator()); return Pair.create(entry.getKey(), pairs); @@ -356,8 +356,7 @@ private KeyRangeIterator rangeMatch(Expression expression, AbstractBounds { - private final Iterator> iterator; + private final Iterator> iterator; private Iterator primaryKeysIterator = CloseableIterator.emptyIterator(); - private ByteComparable.Preencoded byteComparableTerm = null; + private ByteComparable byteComparableTerm = null; - public AllTermsIterator(Iterator> iterator) + public AllTermsIterator(Iterator> iterator) { this.iterator = iterator; } diff --git a/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java b/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java index 91c322ed1282..421b5f52b80d 100644 --- a/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java +++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemtableIndex.java @@ -480,17 +480,17 @@ private ByteComparable encode(ByteBuffer input) * @return iterator of indexed term to primary keys mapping in sorted by indexed term and primary key. */ @Override - public Iterator>> iterator(DecoratedKey min, DecoratedKey max) + public Iterator>> iterator(DecoratedKey min, DecoratedKey max) { int minSubrange = min == null ? 0 : boundaries.getShardForKey(min); int maxSubrange = max == null ? rangeIndexes.length - 1 : boundaries.getShardForKey(max); - List>>> rangeIterators = new ArrayList<>(maxSubrange - minSubrange + 1); + List>>> rangeIterators = new ArrayList<>(maxSubrange - minSubrange + 1); for (int i = minSubrange; i <= maxSubrange; i++) rangeIterators.add(rangeIndexes[i].iterator()); return MergeIterator.get(rangeIterators, - (o1, o2) -> ByteComparable.compare(o1.left, o2.left), + (o1, o2) -> ByteComparable.compare(o1.left, o2.left, TypeUtil.BYTE_COMPARABLE_VERSION), new PrimaryKeysMergeReducer(rangeIterators.size())); } @@ -505,12 +505,12 @@ public Iterator>> iterator * appears at most once per shard, and each key will only be found in a given shard, so there are no values to aggregate; * we simply combine and sort the primary keys from each shard that contains the term. */ - private static class PrimaryKeysMergeReducer extends Reducer>, Pair>> + private static class PrimaryKeysMergeReducer extends Reducer>, Pair>> { - private final Pair>[] rangeIndexEntriesToMerge; + private final Pair>[] rangeIndexEntriesToMerge; private final Comparator comparator; - private ByteComparable.Preencoded term; + private ByteComparable term; @SuppressWarnings("unchecked") // The size represents the number of range indexes that have been selected for the merger @@ -523,7 +523,7 @@ private static class PrimaryKeysMergeReducer extends Reducer> termPair) + public void reduce(int index, Pair> termPair) { Preconditions.checkArgument(rangeIndexEntriesToMerge[index] == null, "Terms should be unique in the memory index"); @@ -534,7 +534,7 @@ public void reduce(int index, Pair> getReduced() + public Pair> getReduced() { Preconditions.checkArgument(term != null, "The term must exist in the memory index"); diff --git a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java index 282bac255fdb..ff195c0e7d97 100644 --- a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java +++ b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java @@ -45,10 +45,8 @@ import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.ComplexColumnData; import org.apache.cassandra.index.sai.IndexContext; -import org.apache.cassandra.index.sai.disk.format.IndexComponentType; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.plan.Expression; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.utils.ByteBufferUtil; @@ -124,6 +122,24 @@ public static ByteBuffer max(ByteBuffer a, ByteBuffer b, AbstractType type, V return a == null ? b : (b == null || compare(b, a, type, version) < 0) ? a : b; } + /** + * Returns the lesser of two {@code ByteComparable} values, based on the result of {@link + * ByteComparable#compare(ByteComparable, ByteComparable, ByteComparable.Version)} comparision. + */ + public static ByteComparable min(ByteComparable a, ByteComparable b) + { + return a == null ? b : (b == null || ByteComparable.compare(b, a, BYTE_COMPARABLE_VERSION) > 0) ? a : b; + } + + /** + * Returns the greater of two {@code ByteComparable} values, based on the result of {@link + * ByteComparable#compare(ByteComparable, ByteComparable, ByteComparable.Version)} comparision. + */ + public static ByteComparable max(ByteComparable a, ByteComparable b) + { + return a == null ? b : (b == null || ByteComparable.compare(b, a, BYTE_COMPARABLE_VERSION) < 0) ? a : b; + } + /** * Returns the value length for the given {@link AbstractType}, selecting 16 for types * that officially use VARIABLE_LENGTH but are, in fact, of a fixed length. @@ -645,9 +661,4 @@ public static ByteBuffer decodeDecimal(ByteBuffer value) var peekableValue = ByteSource.peekable(ByteSource.preencoded(value)); return DecimalType.instance.fromComparableBytes(peekableValue, BYTE_COMPARABLE_VERSION); } - - public static ByteComparable.Version byteComparableVersionForTermsData() - { - return Version.latest().byteComparableVersionFor(IndexComponentType.TERMS_DATA, SSTableFormat.Type.current().info.getLatestVersion()); - } } diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java index 353e2532efa5..0f40b335491b 100644 --- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java +++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteComparable.java @@ -19,7 +19,11 @@ package org.apache.cassandra.utils.bytecomparable; import java.nio.ByteBuffer; +import java.util.Arrays; +import com.google.common.primitives.UnsignedBytes; + +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FastByteOperations; import com.google.common.annotations.VisibleForTesting; @@ -75,11 +79,6 @@ default byte[] asByteComparableArray(Version version) return ByteSourceInverse.readBytes(asComparableBytes(version)); } - default Preencoded preencode(Version version) - { - return preencoded(version, asByteComparableArray(version)); - } - // Simple factories used for testing @VisibleForTesting @@ -99,60 +98,48 @@ static ByteComparable of(int value) return v -> ByteSource.of(value); } - interface Preencoded extends ByteComparable + private static void checkVersion(Version expected, Version actual) { - Version encodingVersion(); - - ByteSource.Duplicatable getPreencodedBytes(); - - @Override - default ByteSource.Duplicatable asComparableBytes(Version version) - { - Preconditions.checkState(version == encodingVersion(), - "Preencoded byte-source at version %s queried at version %s", - encodingVersion(), - version); - return getPreencodedBytes(); - } - - @Override - default ByteSource.Peekable asPeekableBytes(Version version) - { - return asComparableBytes(version); - } - - @Override - default byte[] asByteComparableArray(Version version) - { - return asComparableBytes(version).remainingBytesToArray(); - } + Preconditions.checkState(actual == expected, + "Preprocessed byte-source at version %s queried at version %s", + actual, + expected); } /** * A ByteComparable value that is already encoded for a specific version. Requesting the source with a different * version will result in an exception. */ - static Preencoded preencoded(Version version, ByteBuffer bytes) + static ByteComparable preencoded(Version version, ByteBuffer bytes) { - return new PreencodedByteComparable.Buffer(version, bytes); + return v -> { + checkVersion(version, v); + return ByteSource.preencoded(bytes); + }; } /** * A ByteComparable value that is already encoded for a specific version. Requesting the source with a different * version will result in an exception. */ - static Preencoded preencoded(Version version, byte[] bytes) + static ByteComparable preencoded(Version version, byte[] bytes) { - return new PreencodedByteComparable.Array(version, bytes); + return v -> { + checkVersion(version, v); + return ByteSource.preencoded(bytes); + }; } /** * A ByteComparable value that is already encoded for a specific version. Requesting the source with a different * version will result in an exception. */ - static Preencoded preencoded(Version version, byte[] bytes, int offset, int len) + static ByteComparable preencoded(Version version, byte[] bytes, int offset, int len) { - return new PreencodedByteComparable.Array(version, bytes, offset, len); + return v -> { + checkVersion(version, v); + return ByteSource.preencoded(bytes, offset, len); + }; } /** @@ -201,22 +188,30 @@ static int length(ByteComparable src, Version version) */ static int compare(ByteComparable bytes1, ByteComparable bytes2, Version version) { - return ByteSource.compare(bytes1.asComparableBytes(version), bytes2.asComparableBytes(version)); + ByteSource s1 = bytes1.asComparableBytes(version); + ByteSource s2 = bytes2.asComparableBytes(version); + + if (s1 == null || s2 == null) + return Boolean.compare(s1 != null, s2 != null); + + while (true) + { + int b1 = s1.next(); + int b2 = s2.next(); + int cmp = Integer.compare(b1, b2); + if (cmp != 0) + return cmp; + if (b1 == ByteSource.END_OF_STREAM) + return 0; + } } /** - * Compare two preencoded byte-comparable values, using their encoding versions. - * - * @return the result of the lexicographic unsigned byte comparison of the byte-comparable representations of the - * two arguments + * Compares two bytecomparable encodings lexicographically */ - static int compare(Preencoded a, Preencoded b) + static int compare(ByteBuffer value1, ByteBuffer value2) { - Preconditions.checkArgument(a.encodingVersion() == b.encodingVersion(), - "Cannot compare preencoded byte-comparables of different versions %s vs %s", - a.encodingVersion(), - b.encodingVersion()); - return ByteSource.compare(a.getPreencodedBytes(), b.getPreencodedBytes()); + return FastByteOperations.compareUnsigned(value1, value2); } /** diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java index 9b73c6d5454b..cba0a16102e6 100644 --- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java +++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java @@ -1020,21 +1020,4 @@ public static Duplicatable duplicatable(ByteSource src) return preencoded(ByteSourceInverse.readBytes(src)); } - - static int compare(ByteSource s1, ByteSource s2) - { - if (s1 == null || s2 == null) - return Boolean.compare(s1 != null, s2 != null); - - while (true) - { - int b1 = s1.next(); - int b2 = s2.next(); - int cmp = Integer.compare(b1, b2); - if (cmp != 0) - return cmp; - if (b1 == END_OF_STREAM) - return 0; - } - } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/PreencodedByteComparable.java b/src/java/org/apache/cassandra/utils/bytecomparable/PreencodedByteComparable.java deleted file mode 100644 index c76d424895ed..000000000000 --- a/src/java/org/apache/cassandra/utils/bytecomparable/PreencodedByteComparable.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.utils.bytecomparable; - -import com.google.common.base.Preconditions; - -import java.nio.ByteBuffer; - -abstract class PreencodedByteComparable implements ByteComparable.Preencoded -{ - private final Version version; - - PreencodedByteComparable(Version version) - { - this.version = version; - } - - @Override - public Version encodingVersion() - { - return version; - } - - static class Array extends PreencodedByteComparable - { - private final byte[] bytes; - private final int offset; - private final int length; - - Array(Version version, byte[] bytes) - { - this(version, bytes, 0, bytes.length); - } - - Array(Version version, byte[] bytes, int offset, int length) - { - super(version); - this.bytes = bytes; - this.offset = offset; - this.length = length; - } - - @Override - public ByteSource.Duplicatable getPreencodedBytes() - { - return ByteSource.preencoded(bytes, offset, length); - } - } - - static class Buffer extends PreencodedByteComparable - { - private final ByteBuffer bytes; - - Buffer(Version version, ByteBuffer bytes) - { - super(version); - this.bytes = bytes; - } - - @Override - public ByteSource.Duplicatable getPreencodedBytes() - { - return ByteSource.preencoded(bytes); - } - } -} diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java index 8004d00554f4..32c2812acbde 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieReadBench.java @@ -117,12 +117,12 @@ public void accept(Byte aByte) @Benchmark public int consumeEntries() { - class Counter implements BiConsumer + class Counter implements BiConsumer { int sum = 0; @Override - public void accept(ByteComparable.Preencoded byteComparable, Byte aByte) + public void accept(ByteComparable byteComparable, Byte aByte) { sum += aByte; } @@ -169,7 +169,7 @@ public int iterateValuesUnordered() public int iterateEntries() { int sum = 0; - for (Map.Entry en : trie.entrySet(direction)) + for (Map.Entry en : trie.entrySet(direction)) sum += en.getValue(); return sum; } diff --git a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java index 643cc1a579b8..7dc6585a45f3 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/tries/InMemoryTrieUnionBench.java @@ -128,7 +128,7 @@ public int iterateValuesUnordered() public int iterateEntries() { int sum = 0; - for (Map.Entry en : trie.entrySet()) + for (Map.Entry en : trie.entrySet()) sum += en.getValue(); return sum; } diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java index 102c3cbc76a3..e9bda9210a38 100644 --- a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java +++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieTestBase.java @@ -655,15 +655,15 @@ static Collection reorderBy(Collection original, Comparator compara return list; } - static - void assertMapEquals(Iterator> it1, Iterator> it2) + static void assertMapEquals(Iterator> it1, + Iterator> it2) { List failedAt = new ArrayList<>(); StringBuilder b = new StringBuilder(); while (it1.hasNext() && it2.hasNext()) { - Map.Entry en1 = it1.next(); - Map.Entry en2 = it2.next(); + Map.Entry en1 = it1.next(); + Map.Entry en2 = it2.next(); b.append(String.format("TreeSet %s:%s\n", asString(en2.getKey()), ByteBufferUtil.bytesToHex(en2.getValue()))); b.append(String.format("Trie %s:%s\n", asString(en1.getKey()), ByteBufferUtil.bytesToHex(en1.getValue()))); if (ByteComparable.compare(en1.getKey(), en2.getKey(), byteComparableVersion) != 0 || ByteBufferUtil.compareUnsigned(en1.getValue(), en2.getValue()) != 0) @@ -671,13 +671,13 @@ void assertMapEquals(Iterator> it1, Iterator en1 = it1.next(); + Map.Entry en1 = it1.next(); b.append(String.format("Trie %s:%s\n", asString(en1.getKey()), ByteBufferUtil.bytesToHex(en1.getValue()))); failedAt.add(en1.getKey()); } while (it2.hasNext()) { - Map.Entry en2 = it2.next(); + Map.Entry en2 = it2.next(); b.append(String.format("TreeSet %s:%s\n", asString(en2.getKey()), ByteBufferUtil.bytesToHex(en2.getValue()))); failedAt.add(en2.getKey()); } diff --git a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java index 8435772bd067..9edd672b82ac 100644 --- a/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java +++ b/test/unit/org/apache/cassandra/db/tries/InMemoryTrieThreadedTest.java @@ -102,7 +102,7 @@ public void testThreaded() throws InterruptedException int count = 0; try (OpOrder.Group group = readOrder.start()) { - for (Map.Entry en : trie.entrySet()) + for (Map.Entry en : trie.entrySet()) { String v = value(en.getKey()); Assert.assertEquals(en.getKey().byteComparableAsString(byteComparableVersion), v, en.getValue()); @@ -385,7 +385,7 @@ public void run() int min = writeProgress.get(); try (OpOrder.Group group = readOrder.start()) { - Iterable> entries = trie.entrySet(); + Iterable> entries = trie.entrySet(); checkEntries("", min, true, checkAtomicity, false, PER_MUTATION, entries); } } @@ -415,7 +415,7 @@ public void run() { ByteComparable key = srcLocal[r.nextInt(srcLocal.length)]; int min = writeProgress.get() / (pkeys.length * PER_MUTATION) * PER_MUTATION; - Iterable> entries; + Iterable> entries; try (OpOrder.Group group = readOrder.start()) { @@ -558,7 +558,7 @@ public void checkEntries(String location, boolean checkAtomicity, boolean checkConsecutiveIds, int PER_MUTATION, - Iterable> entries) + Iterable> entries) { long sum = 0; int count = 0; diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java index 1fd65ce42a67..db2f61370754 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexBuilder.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.function.IntSupplier; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -30,7 +31,6 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.memory.RowMapping; -import org.apache.cassandra.index.sai.utils.TypeUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -64,7 +64,7 @@ public static List buildStringTermsEnum(Version version, int terms, i // This logic feels a bit fragile, but it mimics the way we call unescape in the TrieMemoryIndex // before writing to the on disk format. - var encoded = version.onDiskFormat().encodeForTrie(term, UTF8Type.instance).preencode(TypeUtil.BYTE_COMPARABLE_VERSION); + var encoded = version.onDiskFormat().encodeForTrie(term, UTF8Type.instance); termsEnum.add(new TermsEnum(term, encoded, postingsList)); } return termsEnum; @@ -77,10 +77,10 @@ public static class TermsEnum { // Store the original term to ensure that searching by it is successful final ByteBuffer originalTermBytes; - final ByteComparable.Preencoded byteComparableBytes; + final ByteComparable byteComparableBytes; final IntArrayList postings; - TermsEnum(ByteBuffer originalTermBytes, ByteComparable.Preencoded byteComparableBytes, IntArrayList postings) + TermsEnum(ByteBuffer originalTermBytes, ByteComparable byteComparableBytes, IntArrayList postings) { this.originalTermBytes = originalTermBytes; this.byteComparableBytes = byteComparableBytes; @@ -91,7 +91,7 @@ public static class TermsEnum /** * Adds default frequency of 1 to postings */ - static Pair> toTermWithFrequency(TermsEnum te) + static Pair> toTermWithFrequency(TermsEnum te) { return Pair.create(te.byteComparableBytes, Arrays.stream(te.postings.toArray()).boxed() .map(p -> new RowMapping.RowIdWithFrequency(p, 1)) diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java index fd92cff0d546..97277da2490a 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java @@ -175,7 +175,7 @@ private void testTermQueries(Version version, int numTerms, int numPostings) thr var iter = termsEnum.stream() .map(InvertedIndexBuilder::toTermWithFrequency) .collect(Collectors.toList()); - for (Pair> pair : iter) + for (Pair> pair : iter) { final byte[] bytes = ByteSourceInverse.readBytes(pair.left.asComparableBytes(VERSION)); try (PostingList actualPostingList = reader.exactMatch(ByteComparable.preencoded(VERSION, bytes), diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java index caff6b88c4b0..ba8f33cd9951 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/ImmutableOneDimPointValuesTest.java @@ -113,12 +113,12 @@ private TermsIterator buildDescTermEnum(int from, int to) final ByteBuffer minTerm = Int32Type.instance.decompose(from); final ByteBuffer maxTerm = Int32Type.instance.decompose(to); - final AbstractGuavaIterator>> iterator = new AbstractGuavaIterator<>() + final AbstractGuavaIterator>> iterator = new AbstractGuavaIterator<>() { private int currentTerm = from; @Override - protected Pair> computeNext() + protected Pair> computeNext() { if (currentTerm <= to) { @@ -129,7 +129,7 @@ protected Pair> c new RowMapping.RowIdWithFrequency(0, 1), new RowMapping.RowIdWithFrequency(1, 1), new RowMapping.RowIdWithFrequency(2, 1)); - return Pair.create(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, term), postings); + return Pair.create(v -> ByteSource.preencoded(term), postings); } }; diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java index 18ce8d84837b..3a867e41d90f 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/KDTreeIndexBuilder.java @@ -66,7 +66,6 @@ import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; -import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -124,14 +123,14 @@ public long count() private final IndexDescriptor indexDescriptor; private final AbstractType type; - private final AbstractGuavaIterator> terms; + private final AbstractGuavaIterator> terms; private final int size; private final int minSegmentRowId; private final int maxSegmentRowId; public KDTreeIndexBuilder(IndexDescriptor indexDescriptor, AbstractType type, - AbstractGuavaIterator> terms, + AbstractGuavaIterator> terms, int size, int minSegmentRowId, int maxSegmentRowId) @@ -150,12 +149,12 @@ KDTreeIndexSearcher flushAndOpen() throws IOException final TermsIterator termEnum = new MemtableTermsIterator(null, null, new AbstractGuavaIterator<>() { @Override - protected Pair> computeNext() + protected Pair> computeNext() { if (!terms.hasNext()) return endOfData(); - Pair pair = terms.next(); + Pair pair = terms.next(); List postings = new ArrayList<>(pair.right.size()); for (int i = 0; i < pair.right.size(); i++) postings.add(new RowMapping.RowIdWithFrequency(pair.right.get(i), 1)); @@ -291,15 +290,15 @@ public static IndexSearcher buildShortSearcher(IndexDescriptor indexDescriptor, * Returns inverted index where each posting list contains exactly one element equal to the terms ordinal number + * given offset. */ - public static AbstractGuavaIterator> singleOrd(Iterator terms, AbstractType type, int segmentRowIdOffset, int size) + public static AbstractGuavaIterator> singleOrd(Iterator terms, AbstractType type, int segmentRowIdOffset, int size) { - return new AbstractGuavaIterator>() + return new AbstractGuavaIterator>() { private long currentTerm = 0; private int currentSegmentRowId = segmentRowIdOffset; @Override - protected Pair computeNext() + protected Pair computeNext() { if (currentTerm++ >= size) { @@ -311,7 +310,7 @@ protected Pair computeNext() assertTrue(terms.hasNext()); final ByteSource encoded = TypeUtil.asComparableBytes(terms.next(), type, TypeUtil.BYTE_COMPARABLE_VERSION); - return Pair.create(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, ByteSourceInverse.readBytes(encoded)), postings); + return Pair.create(v -> encoded, postings); } }; } diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java index 580217271946..27bff7fa40cb 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/kdtree/NumericIndexWriterTest.java @@ -47,7 +47,6 @@ import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; -import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; import org.apache.lucene.index.PointValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; @@ -194,13 +193,13 @@ private TermsIterator buildTermEnum(int startTermInclusive, int endTermExclusive final ByteBuffer minTerm = Int32Type.instance.decompose(startTermInclusive); final ByteBuffer maxTerm = Int32Type.instance.decompose(endTermExclusive); - final AbstractGuavaIterator>> iterator = new AbstractGuavaIterator<>() + final AbstractGuavaIterator>> iterator = new AbstractGuavaIterator<>() { private int currentTerm = startTermInclusive; private int currentRowId = 0; @Override - protected Pair> computeNext() + protected Pair> computeNext() { if (currentTerm >= endTermExclusive) { @@ -210,9 +209,7 @@ protected Pair> c final List postings = new ArrayList<>(); postings.add(new RowMapping.RowIdWithFrequency(currentRowId++, 1)); final ByteSource encoded = Int32Type.instance.asComparableBytes(term, TypeUtil.BYTE_COMPARABLE_VERSION); - byte[] bytes = new byte[4]; - encoded.nextBytes(bytes); - return Pair.create(ByteComparable.preencoded(TypeUtil.BYTE_COMPARABLE_VERSION, bytes), postings); + return Pair.create(v -> encoded, postings); } }; diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java index 453c4e6bdce2..e96eeb10b718 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/v6/TermsDistributionTest.java @@ -42,13 +42,12 @@ public class TermsDistributionTest { - static final ByteComparable.Version VERSION = ByteComparable.Version.OSS41; @Test public void testEmpty() { AbstractType type = Int32Type.instance; - TermsDistribution td = new TermsDistribution.Builder(type, VERSION, 10, 10).build(); + TermsDistribution td = new TermsDistribution.Builder(type, 10, 10).build(); assertEquals(0, td.estimateNumRowsMatchingExact(encode(1))); assertEquals(0, td.estimateNumRowsInRange(encode(0), encode(1000))); } @@ -57,7 +56,7 @@ public void testEmpty() public void testExactMatch() { AbstractType type = Int32Type.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 10, 10); + var builder = new TermsDistribution.Builder(type, 10, 10); for (int i = 0; i < 1000; i++) builder.add(encode(i), 1); var td = builder.build(); @@ -76,7 +75,7 @@ public void testExactMatch() public void testRangeMatch() { AbstractType type = Int32Type.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 10, 10); + var builder = new TermsDistribution.Builder(type, 10, 10); for (int i = 0; i < 1000; i++) builder.add(encode(i), 1); var td = builder.build(); @@ -120,7 +119,7 @@ public void testMostFrequentItems() int frequentCount = 100; // whatever > 1 AbstractType type = Int32Type.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 10, 10); + var builder = new TermsDistribution.Builder(type, 10, 10); for (int i = 0; i < 1000; i++) builder.add(encode(i), (i == frequentValue) ? frequentCount : 1); var td = builder.build(); @@ -152,7 +151,7 @@ public void testFractionalBuckets() // Test if we get reasonable range estimates when selecting a fraction of a single bucket: AbstractType type = DoubleType.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 13, 13); + var builder = new TermsDistribution.Builder(type, 13, 13); var COUNT = 100000; for (int i = 0; i < COUNT; i++) builder.add(encode((double) i / COUNT), 1); @@ -190,7 +189,7 @@ public void testFractionalBucketsBigInt() // Test if we get reasonable range estimates when selecting a fraction of a single bucket: AbstractType type = IntegerType.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 13, 13); + var builder = new TermsDistribution.Builder(type, 13, 13); var COUNT = 100000; for (int i = 0; i < COUNT; i++) builder.add(encodeAsBigInt(i), 1); @@ -219,7 +218,7 @@ public void testFractionalBucketsDecimal() // Test if we get reasonable range estimates when selecting a fraction of a single bucket: AbstractType type = DecimalType.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 13, 13); + var builder = new TermsDistribution.Builder(type, 13, 13); var COUNT = 100000; for (int i = 0; i < COUNT; i++) builder.add(encodeAsDecimal((double) i / COUNT), 1); @@ -254,7 +253,7 @@ public void testFractionalBucketsDecimal() public void testSerde() throws IOException { AbstractType type = DoubleType.instance; - var builder = new TermsDistribution.Builder(type, VERSION, 10, 10); + var builder = new TermsDistribution.Builder(type, 10, 10); var COUNT = 100000; for (int i = 0; i < COUNT; i++) builder.add(encode((double) i / COUNT), 1); diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java index f6b629c5bed5..8864108e784a 100644 --- a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java +++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTestBase.java @@ -328,10 +328,10 @@ private void assertEqualsQuery(int value, int... partitionKeys) assertFalse(result.hasNext()); } - private void assertNextEntryInIterator(Iterator>> iter, int term, int... primaryKeys) + private void assertNextEntryInIterator(Iterator>> iter, int term, int... primaryKeys) { assertTrue(iter.hasNext()); - Pair> entry = iter.next(); + Pair> entry = iter.next(); assertEquals(term, termFromComparable(entry.left)); for (int i = 0; i < primaryKeys.length; i++) {