[WIP] CNDB-15608 port CASSANDRA-18673 to reduce disk usage of row-aware indexes

.gitignore

@@ -90,3 +90,5 @@ venv/
 # build-scripts will put cassandra-builds into the directory
 cassandra-builds/
 cassandra-dtest/
+
+.ai/

CHANGES.txt

@@ -1,4 +1,5 @@
 Future version (tbd)
+ * Reduce size of per-SSTable index components for SAI (CASSANDRA-18673)
  * Require only MODIFY permission on base when updating table with MV (STAR-564)
 Merged from 5.1:
  * Expose current compaction throughput in nodetool (CASSANDRA-13890)

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -429,6 +429,16 @@ public enum CassandraRelevantProperties
      */
     SAI_MONITORING_EXECUTION_INFO_ENABLED("cassandra.sai.monitoring_execution_info_enabled", "true"),
 
+    /**
+     * Used to determine the block size and block mask for the clustering sorted terms.
+     */
+    SAI_SORTED_TERMS_CLUSTERING_BLOCK_SHIFT("cassandra.sai.sorted_terms_clustering_block_shift", "4"),
+
+    /**
+     * Used to determine the block size and block mask for the partition sorted terms.
+     */
+    SAI_SORTED_TERMS_PARTITION_BLOCK_SHIFT("cassandra.sai.sorted_terms_partition_block_shift", "4"),
+
     /** Whether vector type only allows float vectors. True by default. **/
     VECTOR_FLOAT_ONLY("cassandra.float_only_vectors", "true"),
     /** Enables use of vector type. True by default. **/

src/java/org/apache/cassandra/db/marshal/AbstractType.java

@@ -178,6 +178,11 @@ public <V> T compose(V value, ValueAccessor<V> accessor)
         return getSerializer().deserialize(value, accessor);
     }
 
+    public ByteBuffer decomposeUntyped(Object value)
+    {
+        return decompose((T) value);
+    }
+
     public ByteBuffer decompose(T value)
     {
         return getSerializer().serialize(value);

src/java/org/apache/cassandra/index/sai/SSTableContext.java

@@ -145,7 +145,7 @@ public PrimaryKeyMap.Factory primaryKeyMapFactory()
      */
     public int openFilesPerSSTable()
     {
-        return perSSTableComponents.onDiskFormat().openFilesPerSSTable();
+        return perSSTableComponents.onDiskFormat().openFilesPerSSTable(sstable.hasClustering);
     }
 
     @Override

src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java

@@ -29,6 +29,7 @@
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.ThreadSafe;
 
@@ -265,7 +266,7 @@ public StorageAttachedIndexQueryPlan queryPlanFor(RowFilter rowFilter)
     @Override
     public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata, long keyCount)
     {
-        IndexDescriptor indexDescriptor = IndexDescriptor.empty(descriptor);
+        IndexDescriptor indexDescriptor = IndexDescriptor.empty(descriptor, tableMetadata);
         try
         {
             return new StorageAttachedIndexWriter(indexDescriptor, tableMetadata, indices, tracker, keyCount, baseCfs.metric);
@@ -290,7 +291,7 @@ public boolean handles(IndexTransaction.Type type)
     @Override
     public Set<Component> componentsForNewSSTable()
     {
-        return IndexDescriptor.componentsForNewlyFlushedSSTable(indices, version);
+        return IndexDescriptor.componentsForNewlyFlushedSSTable(indices, version, baseCfs.metadata().comparator.size() > 0);
     }
 
     @Override

src/java/org/apache/cassandra/index/sai/disk/PerSSTableWriter.java

@@ -21,6 +21,7 @@
 
 import com.google.common.base.Stopwatch;
 
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
 
 /**
@@ -33,6 +34,9 @@ public interface PerSSTableWriter
     default void startPartition(long position) throws IOException
     {}
 
+    default void startPartition(DecoratedKey decoratedKey) throws IOException
+    {}
+
     void nextRow(PrimaryKey primaryKey) throws IOException;
 
     default void complete(Stopwatch stopwatch) throws IOException

src/java/org/apache/cassandra/index/sai/disk/PostingListKeyRangeIterator.java

@@ -140,11 +140,11 @@ public void close() throws IOException
 
             FileUtils.closeQuietly(postingList, primaryKeyMap);
         }
-        else {
+        else
+        {
             logger.warn("PostingListKeyRangeIterator is already closed",
                         new IllegalStateException("PostingListKeyRangeIterator is already closed"));
         }
-
     }
 
     private boolean exhausted()

src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java

@@ -38,7 +38,7 @@ public interface PrimaryKeyMap extends Closeable
      * A factory for creating {@link PrimaryKeyMap} instances. Implementations of this
      * interface are expected to be threadsafe.
      */
-    public interface Factory extends Closeable
+    interface Factory extends Closeable
     {
         /**
          * Creates a new {@link PrimaryKeyMap} instance
@@ -121,7 +121,7 @@ default PrimaryKey primaryKeyFromRowId(long sstableRowId, @Nonnull PrimaryKey lo
      * @param key the {@link PrimaryKey} to lookup
      * @return an sstable row id or a negative value if no row is found
      */
-    long ceiling(PrimaryKey key);
+   long ceiling(PrimaryKey key);
 
     /**
      * Returns the sstable row id associated with the greatest {@link PrimaryKey} less than or equal to the given
@@ -132,7 +132,7 @@ default PrimaryKey primaryKeyFromRowId(long sstableRowId, @Nonnull PrimaryKey lo
      * @param key the {@link PrimaryKey} to lookup
      * @return an sstable row id or a negative value if no row is found
      */
-    long floor(PrimaryKey key);
+   long floor(PrimaryKey key);
 
     /**
      * Returns the number of primary keys in the map

src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java

@@ -131,7 +131,10 @@ public void startPartition(DecoratedKey key, long position)
 
         try
         {
+            // Implemented in v1
             perSSTableWriter.startPartition(position);
+            // Implemented in v2
+            perSSTableWriter.startPartition(key);
         }
         catch (Throwable t)
         {

src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java

@@ -26,6 +26,7 @@
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.Throwables;
 import org.apache.lucene.store.ChecksumIndexInput;
 
 public interface IndexComponent
@@ -50,10 +51,10 @@ interface ForRead extends IndexComponent
         @Override
         IndexComponents.ForRead parent();
 
-        FileHandle createFileHandle();
+        FileHandle createFileHandle(Throwables.DiscreteAction<?> cleanup);
 
         /**
-         * Opens a file handle for the provided index component similarly to {@link #createFileHandle()},
+         * Opens a file handle for the provided index component similarly to {@link #createFileHandle(Throwables.DiscreteAction)},
          * but this method shoud be called instead of the aforemented one if the access is done during index building, that is
          * before the full index that this is a part of has been finalized.
          * <p>

src/java/org/apache/cassandra/index/sai/disk/format/IndexComponentType.java

@@ -82,24 +82,36 @@ public enum IndexComponentType
      * V1
      */
     OFFSETS_VALUES("OffsetsValues"),
+
     /**
-     * An on-disk trie containing the primary keys used for looking up the rowId from a partition key
-     *
-     * V2
+     * An on-disk block-packed index containing the starting and ending rowIds for each partition.
      */
-    PRIMARY_KEY_TRIE("PrimaryKeyTrie"),
+    PARTITION_SIZES("PartitionSizes"),
+
     /**
-     * Prefix-compressed blocks of primary keys used for rowId to partition key lookups
-     *
+     * Prefix-compressed blocks of partition keys used for rowId to partition key lookups
+     * <p>
      * V2
      */
-    PRIMARY_KEY_BLOCKS("PrimaryKeyBlocks"),
+    PARTITION_KEY_BLOCKS("PartitionKeyBlocks"),
+
     /**
-     * Encoded sequence of offsets to primary key blocks
-     *
+     * Encoded sequence of offsets to partition key blocks
+     * <p>
      * V2
      */
-    PRIMARY_KEY_BLOCK_OFFSETS("PrimaryKeyBlockOffsets"),
+    PARTITION_KEY_BLOCK_OFFSETS("PartitionKeyBlockOffsets"),
+
+    /**
+     * Prefix-compressed blocks of clustering keys used for rowId to clustering key lookups
+     */
+    CLUSTERING_KEY_BLOCKS("ClusteringKeyBlocks"),
+
+    /**
+     * Encoded sequence of offsets to clustering key blocks
+     */
+    CLUSTERING_KEY_BLOCK_OFFSETS("ClusteringKeyBlockOffsets"),
+
     /**
      * Stores per-sstable metadata.
      *

src/java/org/apache/cassandra/index/sai/disk/format/IndexComponents.java

@@ -30,6 +30,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.index.sai.IndexContext;
 import org.apache.cassandra.io.sstable.Component;
@@ -183,9 +184,13 @@ default Set<IndexComponentType> expectedComponentsForVersion()
     {
         return isPerIndexGroup()
                ? onDiskFormat().perIndexComponentTypes(context())
-               : onDiskFormat().perSSTableComponentTypes();
+               : onDiskFormat().perSSTableComponentTypes(hasClustering());
     }
 
+    boolean hasClustering();
+
+    ClusteringComparator comparator();
+
     default ByteComparable.Version byteComparableVersionFor(IndexComponentType component)
     {
         return version().byteComparableVersionFor(component, descriptor().version);